diff options
2655 files changed, 27779 insertions, 19508 deletions
diff --git a/.gitignore b/.gitignore index 0f289299ac8..14366c5c31b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ Testing *_test_app /hadoop/dependency-reduced-pom.xml /vespa-hadoop/dependency-reduced-pom.xml +.preprocessed/ +install_manifest.txt diff --git a/bootstrap.sh b/bootstrap.sh index 3a4625cbaf0..4e3761649e0 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -12,12 +12,11 @@ elif [ "$1" = "full" ]; then # Build all java modules required by C++ testing MODE=full elif [ "$1" = "java" ]; then - # Build minial set of java modules, then run mvn install with arguments $2, - # $3, etc. + # Build only plugins MODE=java elif [ "$1" = "default" ]; then MODE=default -elif [ "$1" = "-h" -o "$1" = "--help" ]; then +elif [ "$1" = "-h" ] || [ "$1" = "--help" ]; then usage exit 0 else @@ -36,53 +35,30 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map # NOTES ON BUILDING JAVA MODULES # -# When building a random module like orchestrator, mvn lists 2 plugins as -# prerequisites in the Reactor summary: bundle-plugin and config-class-plugin. +# mvn is unable to resolve references to a plugin, if the same mvn +# program builds the plugin in the same reactor build. # -# It appears mvn is unable to resolve references to a plugin, if the same mvn -# program builds the plugin, e.g. getting missing class errors. +# Therefore, we need to manually build all plugins first. # -# Therefore, we need to manually build all plugins first, then ensure we build -# the rest of module without rebuilding the plugins (mvn's -rf). To be more -# exact, mvn displays the order of modules it will build in the Reactor summary -# - we need to build all modules up to and including all plugins in that list: -# . (i.e. parent), annotations, scalalib, bundle-plugin, yolean, vespajlib, -# configgen, and config-class-plugin. By using mvn -rf we can combine the -# building of several of these into fewer commands to save time. -# -# After bootstrapping, the following mvn command must use -rf to avoid building -# the plugins again: The 'java' mode runs mvn install with the correct -rf. So -# to bootstrap the building of the orchestrator modules and all of its -# dependencies, do: 'bootstrap.sh java -pl orchestrator'. To build everything -# do 'bootstrap.sh java'. +# The 'java' mode only builds the plugins. +# The 'default' mode also builds some modules needed by C++ code. +# The 'full' mode also builds modules needed by C++ tests. -# Note: Why not just 'mvn_install -am -pl bundle-plugin'? Because on -# Screwdriver (not locally), mvn fails to resolve bundle-plugin while parsing -# pom files recursivelly. Therefore, shut off recursiveness -N until -# bundle-plugin is built. -MODULES=" - . - annotations - scalalib - bundle-plugin - " +# must install parent pom first: +mvn_install -N -for module in $MODULES; do - (cd $module && mvn_install -N) -done +# and build plugins first: +mvn_install -f maven-plugins/pom.xml -mvn_install -am -rf configgen -pl config-class-plugin +# now everything else should just work with normal maven dependency resolution: case "$MODE" in java) - mvn_install -am -rf yolean -pl vespajlib - shift - mvn_install -rf config-lib -am "$@" ;; full) - mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus -rf yolean + mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus ;; default) - mvn_install -am -pl filedistributionmanager -rf yolean + mvn_install -am -pl filedistributionmanager ;; esac diff --git a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java index f9fbb21d117..c01783428b4 100644 --- a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java +++ b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java @@ -17,14 +17,18 @@ import java.util.Collection; import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; +import java.util.zip.ZipEntry; import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; -import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertTrue; /** * Verifies the bundle jar file built and its manifest. @@ -101,12 +105,23 @@ public class BundleIT { public void require_that_manifest_contains_bundle_class_path() { String bundleClassPath = mainAttributes.getValue("Bundle-ClassPath"); assertThat(bundleClassPath, containsString(".,")); - assertThat(bundleClassPath, containsString("dependencies/jrt-6-SNAPSHOT.jar")); + // If bundle-plugin-test is compiled in a mvn command that also built jrt, + // the jrt artifact is jrt.jar, otherwise the installed and versioned artifact + // is used: jrt-6-SNAPSHOT.jar. + assertThat(bundleClassPath, anyOf( + containsString("dependencies/jrt-6-SNAPSHOT.jar"), + containsString("dependencies/jrt.jar"))); } @Test public void require_that_component_jar_file_contains_compile_artifacts() { - assertNotNull(jarFile.getEntry("dependencies/jrt-6-SNAPSHOT.jar")); + ZipEntry versionedEntry = jarFile.getEntry("dependencies/jrt-6-SNAPSHOT.jar"); + ZipEntry unversionedEntry = jarFile.getEntry("dependencies/jrt.jar"); + if (versionedEntry == null) { + assertNotNull(unversionedEntry); + } else { + assertNull(unversionedEntry); + } } @@ -134,6 +149,8 @@ public class BundleIT { assertThat(mainBundleClassPaths, hasItems( endsWith("target/classes"), - allOf(containsString("jrt"), containsString(".jar"), containsString("m2/repository")))); + anyOf( + allOf(containsString("jrt"), containsString(".jar"), containsString("m2/repository")), + containsString("jrt/target/jrt.jar")))); } } diff --git a/chain/.gitignore b/chain/.gitignore index c0dd71fac68..16ab3954e1b 100644 --- a/chain/.gitignore +++ b/chain/.gitignore @@ -1,4 +1,3 @@ chain.iml target - /pom.xml.build diff --git a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java index 77e492395ce..5ae2765c074 100644 --- a/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java +++ b/chain/src/main/java/com/yahoo/component/chain/dependencies/ordering/ComponentNameProvider.java @@ -48,7 +48,8 @@ class ComponentNameProvider extends NameProvider { return 1; } - public @Override String toString() { + @Override + public String toString() { StringBuilder b=new StringBuilder("components ["); for (@SuppressWarnings("rawtypes") Iterator<ComponentNode> i=nodes.iterator(); i.hasNext(); ) { diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 053f4efd2a7..42af701ac63 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -27,6 +27,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-library</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> </dependency> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 2a9c1e0e9b3..fed4bac3b02 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -348,6 +348,22 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd newStates.add(state); metricUpdater.updateClusterStateMetrics(cluster, state); systemStateBroadcaster.handleNewSystemState(state); + // Iff master, always store new version in ZooKeeper _before_ publishing to any + // nodes so that a cluster controller crash after publishing but before a successful + // ZK store will not risk reusing the same version number. + if (masterElectionHandler.isMaster()) { + storeClusterStateVersionToZooKeeper(state); + } + } + + private void storeClusterStateVersionToZooKeeper(ClusterState state) { + try { + database.saveLatestSystemStateVersion(databaseContext, state.getVersion()); + } catch (InterruptedException e) { + // Rethrow as RuntimeException to propagate exception up to main thread method. + // Don't want to hide failures to write cluster state version. + throw new RuntimeException("ZooKeeper write interrupted", e); + } } /** diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java index df0d2f78c61..c9d9645735c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java @@ -6,10 +6,13 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; +import com.yahoo.vespa.clustercontroller.core.hostinfo.Metrics; import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNode; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import java.util.List; +import java.util.Optional; /** * Checks if a node can be upgraded. @@ -17,6 +20,7 @@ import java.util.List; * @author dybis */ public class NodeStateChangeChecker { + public static final String BUCKETS_METRIC_NAME = "vds.datastored.alldisks.buckets"; private final int minStorageNodesUp; private double minRatioOfStorageNodesUp; @@ -95,6 +99,11 @@ public class NodeStateChangeChecker { "Requested node type: " + node.getType().toString()); } + NodeInfo nodeInfo = clusterInfo.getStorageNodeInfo(node.getIndex()); + if (nodeInfo == null) { + return Result.createDisallowed("Unknown node " + node); + } + // If the new state and description equals the existing, we're done. This is done for 2 cases: // - We can short-circuit setting of a new wanted state, which e.g. hits ZooKeeper. // - We ensure that clients that have previously set the wanted state, continue @@ -106,35 +115,73 @@ public class NodeStateChangeChecker { switch (newState.getState()) { case UP: - return canSetStateUp(node, oldState.getState()); + return canSetStateUp(nodeInfo, oldState.getState()); case MAINTENANCE: - return canSetStateMaintenance(node, clusterState); + return canSetStateMaintenanceTemporarily(node, clusterState); + case DOWN: + return canSetStateDownPermanently(nodeInfo, clusterState); default: - return Result.createDisallowed("Safe only supports state UP and MAINTENANCE, you tried: " + newState); + return Result.createDisallowed("Destination node state unsupported in safe mode: " + newState); } } - private Result canSetStateUp(Node node, State oldState) { + private Result canSetStateDownPermanently(NodeInfo nodeInfo, ClusterState clusterState) { + State reportedState = nodeInfo.getReportedState().getState(); + if (reportedState != State.UP) { + return Result.createDisallowed("Reported state (" + reportedState + + ") is not UP, so no bucket data is available"); + } + + State currentState = clusterState.getNodeState(nodeInfo.getNode()).getState(); + if (currentState != State.RETIRED) { + return Result.createDisallowed("Only retired nodes are allowed to be set to DOWN in safe mode - is " + + currentState); + } + + Result thresholdCheckResult = checkUpThresholds(clusterState); + if (!thresholdCheckResult.settingWantedStateIsAllowed()) { + return thresholdCheckResult; + } + + HostInfo hostInfo = nodeInfo.getHostInfo(); + Integer hostInfoNodeVersion = hostInfo.getClusterStateVersionOrNull(); + int clusterControllerVersion = clusterState.getVersion(); + if (hostInfoNodeVersion == null || hostInfoNodeVersion != clusterControllerVersion) { + return Result.createDisallowed("Cluster controller at version " + clusterControllerVersion + + " got info for storage node " + nodeInfo.getNodeIndex() + " at a different version " + + hostInfoNodeVersion); + } + + Optional<Metrics.Value> bucketsMetric = hostInfo.getMetrics().getValue(BUCKETS_METRIC_NAME); + if (!bucketsMetric.isPresent() || bucketsMetric.get().getLast() == null) { + return Result.createDisallowed("Missing last value of the " + BUCKETS_METRIC_NAME + + " metric for storage node " + nodeInfo.getNodeIndex()); + } + + long lastBuckets = bucketsMetric.get().getLast(); + if (lastBuckets > 0) { + return Result.createDisallowed("The storage node manages " + lastBuckets + " buckets"); + } + + return Result.allowSettingOfWantedState(); + } + + private Result canSetStateUp(NodeInfo nodeInfo, State oldState) { if (oldState != State.MAINTENANCE) { return Result.createDisallowed("Refusing to set wanted state to up when it is currently in " + oldState); } - if (clusterInfo.getNodeInfo(node).getReportedState().getState() != State.UP) { + if (nodeInfo.getReportedState().getState() != State.UP) { return Result.createDisallowed("Refuse to set wanted state to UP, " + "since the reported state is not UP (" + - clusterInfo.getNodeInfo(node).getReportedState().getState() + ")"); + nodeInfo.getReportedState().getState() + ")"); } return Result.allowSettingOfWantedState(); } - private Result canSetStateMaintenance(Node node, ClusterState clusterState) { - NodeInfo nodeInfo = clusterInfo.getNodeInfo(node); - if (nodeInfo == null) { - return Result.createDisallowed("Unknown node " + node); - } - NodeState currentState = clusterState.getNodeState(node); - if (currentState.getState() == State.DOWN) { + private Result canSetStateMaintenanceTemporarily(Node node, ClusterState clusterState) { + if (clusterState.getNodeState(node).getState() == State.DOWN) { return Result.allowSettingOfWantedState(); } @@ -148,13 +195,9 @@ public class NodeStateChangeChecker { return ongoingChanges; } - if (clusterInfo.getStorageNodeInfo().size() < minStorageNodesUp) { - return Result.createDisallowed("There are only " + clusterInfo.getStorageNodeInfo().size() + - " storage nodes up, while config requires at least " + minStorageNodesUp); - } - Result fractionCheck = isFractionHighEnough(clusterState); - if (!fractionCheck.settingWantedStateIsAllowed()) { - return fractionCheck; + Result thresholdCheckResult = checkUpThresholds(clusterState); + if (!thresholdCheckResult.settingWantedStateIsAllowed()) { + return thresholdCheckResult; } return Result.allowSettingOfWantedState(); @@ -182,17 +225,22 @@ public class NodeStateChangeChecker { return upNodesCount; } - private Result isFractionHighEnough(ClusterState clusterState) { + private Result checkUpThresholds(ClusterState clusterState) { + if (clusterInfo.getStorageNodeInfo().size() < minStorageNodesUp) { + return Result.createDisallowed("There are only " + clusterInfo.getStorageNodeInfo().size() + + " storage nodes up, while config requires at least " + minStorageNodesUp); + } + final int nodesCount = clusterInfo.getStorageNodeInfo().size(); final int upNodesCount = contentNodesWithAvailableNodeState(clusterState); if (nodesCount == 0) { - return Result.createDisallowed("No storage nodes in cluster state, not safe to restart."); + return Result.createDisallowed("No storage nodes in cluster state"); } if (((double)upNodesCount) / nodesCount < minRatioOfStorageNodesUp) { - return Result.createDisallowed("Not enough storage nodes running, running: " + upNodesCount - + " total storage nodes " + nodesCount + - " required fraction " + minRatioOfStorageNodesUp); + return Result.createDisallowed("Not enough storage nodes running: " + upNodesCount + + " of " + nodesCount + " storage nodes are up which is less that the required fraction of " + + minRatioOfStorageNodesUp); } return Result.allowSettingOfWantedState(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 0f3bc6ada88..33d8ad27738 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -110,11 +110,7 @@ public class SystemStateBroadcaster { if (systemState == null) return false; List<NodeInfo> recipients = resolveStateVersionSendSet(dbContext); - // Store new version in ZooKeeper _before_ publishing to any nodes so that a - // cluster controller crash after publishing but before a successful ZK store - // will not risk reusing the same version number. - if (!recipients.isEmpty() && !systemState.isOfficial()) { - database.saveLatestSystemStateVersion(dbContext, systemState.getVersion()); + if (!systemState.isOfficial()) { systemState.setOfficial(true); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/config/.gitignore b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/config/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/config/.gitignore +++ /dev/null diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java index b2924516f26..4770bb40545 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Keeper for Metrics for HostInfo. @@ -13,7 +14,17 @@ import java.util.List; */ public class Metrics { - public List<Metric> getValues() { return Collections.unmodifiableList(metricsList); } + public Optional<Value> getValue(String name) { + for (Metric metric : metricsList) { + if (name.equals(metric.getName())) { + return Optional.ofNullable(metric.getValue()); + } + } + + return Optional.empty(); + } + + public List<Metric> getMetrics() { return Collections.unmodifiableList(metricsList); } public static class Metric { private final String name; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java index 8958bce8ccd..308b189841d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java @@ -39,7 +39,7 @@ public class PartitionStateRequest extends Request<Response.PartitionResponse> { } private static void fillInMetrics(Metrics metrics, Response.PartitionResponse result) { - for (Metrics.Metric metric: metrics.getValues()) { + for (Metrics.Metric metric: metrics.getMetrics()) { fillInMetricValue(metric.getName(), metric.getValue(), result); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index 2b37d29e2ec..355166f6a8d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -12,7 +12,6 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import org.junit.After; import org.junit.Ignore; import org.junit.Test; @@ -27,17 +26,8 @@ public class DatabaseTest extends FleetControllerTest { private static Logger log = Logger.getLogger(DatabaseTest.class.getName()); - protected Supervisor supervisor; - - @After - public void tearDown() throws Exception { - if (supervisor != null) { - supervisor.transport().shutdown().join(); - } - super.tearDown(); - } - - private void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) { + // Note: different semantics than FleetControllerTest.setWantedState + protected void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) { int rpcPort = fleetController.getRpcPort(); if (supervisor == null) { supervisor = new Supervisor(new Transport()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index d0aa0bceba9..59e850e0485 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -1,6 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.log.LogLevel; @@ -10,6 +16,7 @@ import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; @@ -26,6 +33,7 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; @@ -38,13 +46,14 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * @author humbe + * @author Håkon Humberset */ public abstract class FleetControllerTest implements Waiter { private static Logger log = Logger.getLogger(FleetControllerTest.class.getName()); protected static final int DEFAULT_NODE_COUNT = 10; + protected Supervisor supervisor; protected FakeTimer timer = new FakeTimer(); protected boolean usingFakeTimer = false; protected Slobrok slobrok; @@ -276,6 +285,9 @@ public abstract class FleetControllerTest implements Waiter { System.err.println("STOPPING TEST " + testName); testName = null; } + if (supervisor != null) { + supervisor.transport().shutdown().join(); + } if (fleetController != null) { fleetController.shutdown(); fleetController = null; @@ -504,4 +516,23 @@ public abstract class FleetControllerTest implements Waiter { .collect(Collectors.toSet()); } + protected void setWantedState(DummyVdsNode node, State state, String reason) { + if (supervisor == null) { + supervisor = new Supervisor(new Transport()); + } + NodeState ns = new NodeState(node.getType(), state); + if (reason != null) ns.setDescription(reason); + Target connection = supervisor.connect(new Spec("localhost", fleetController.getRpcPort())); + Request req = new Request("setNodeState"); + req.parameters().add(new StringValue(node.getSlobrokName())); + req.parameters().add(new StringValue(ns.serialize())); + connection.invokeSync(req, timeoutS); + if (req.isError()) { + assertTrue("Failed to invoke setNodeState(): " + req.errorCode() + ": " + req.errorMessage(), false); + } + if (!req.checkReturnTypes("s")) { + assertTrue("Failed to invoke setNodeState(): Invalid return types.", false); + } + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index d4cb988a986..a298be2a035 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -9,6 +9,7 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.log.LogLevel; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.State; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -20,8 +21,10 @@ import java.util.List; import java.util.concurrent.TimeoutException; import java.util.logging.Logger; +import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; public class MasterElectionTest extends FleetControllerTest { @@ -440,4 +443,56 @@ public class MasterElectionTest extends FleetControllerTest { waitForMaster(1); } + /** + * Should always write new version to ZooKeeper, even if the version will not + * be published to any nodes. External services may still observe the version + * number via the cluster REST API, and we should therefore ensure that we never + * risk rolling back the version number in the face of a reelection. + */ + @Test + public void cluster_state_version_written_to_zookeeper_even_with_empty_send_set() throws Exception { + startingTest("MasterElectionTest::cluster_state_version_written_to_zookeeper_even_with_empty_send_set"); + FleetControllerOptions options = new FleetControllerOptions("mycluster"); + options.masterZooKeeperCooldownPeriod = 1; + options.minRatioOfDistributorNodesUp = 0; + options.minRatioOfStorageNodesUp = 0; + options.minDistributorNodesUp = 0; + options.minStorageNodesUp = 1; + setUpFleetController(3, true, options); + setUpVdsNodes(true, new DummyVdsNodeOptions()); + fleetController = fleetControllers.get(0); // Required to prevent waitForStableSystem from NPE'ing + waitForStableSystem(); + waitForMaster(0); + + // Explanation for this convoluted sequence of actions: we want to trigger a scenario where + // we have a cluster state version bump _without_ there being any nodes to send the new state + // to. If there's an "optimization" to skip writing the version to ZooKeeper if there are no + // nodes in the version send set, a newly elected, different master will end up reusing the + // very same version number. + // We mark all nodes' Reported states as down (which means an empty send set, as no nodes are + // online), then mark one storage node as Wanted state as Maintenance. This forces a cluster + // state change. + this.nodes.forEach(n -> { + n.disconnectImmediately(); + waitForCompleteCycle(0); + }); + setWantedState(this.nodes.get(2*10 - 1), State.MAINTENANCE, "bar"); + waitForCompleteCycle(0); + + // This receives the version number of the highest _working_ cluster state, with + // no guarantees that it has been published to any nodes yet. + final long preElectionVersionNumber = fleetControllers.get(0).getSystemState().getVersion(); + + // Nuke controller 1, leaving controller 1 in charge. + // It should have observed the most recently written version number and increase this + // number before publishing its own new state. + fleetControllers.get(0).shutdown(); + waitForMaster(1); + waitForCompleteCycle(1); + + final long postElectionVersionNumber = fleetControllers.get(1).getSystemState().getVersion(); + + assertThat(postElectionVersionNumber, greaterThan(preElectionVersionNumber)); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java index 5b53e524102..0d2f12dfdcd 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java @@ -20,9 +20,13 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import static org.hamcrest.core.StringContains.containsString; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,8 +40,9 @@ public class NodeStateChangeCheckerTest { private static final Node nodeDistributor = new Node(NodeType.DISTRIBUTOR, 1); private static final Node nodeStorage = new Node(NodeType.STORAGE, 1); - private static final NodeState upNodeState = new NodeState(NodeType.STORAGE, State.UP); - public static final NodeState maintenanceNodeState = createNodeState(State.MAINTENANCE, "Orchestrator"); + private static final NodeState UP_NODE_STATE = new NodeState(NodeType.STORAGE, State.UP); + public static final NodeState MAINTENANCE_NODE_STATE = createNodeState(State.MAINTENANCE, "Orchestrator"); + public static final NodeState DOWN_NODE_STATE = createNodeState(State.DOWN, "RetireEarlyExpirer"); private static NodeState createNodeState(State state, String description) { return new NodeState(NodeType.STORAGE, state).setDescription(description); @@ -122,17 +127,30 @@ public class NodeStateChangeCheckerTest { NodeState newState = new NodeState(NodeType.STORAGE, State.INITIALIZING); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), SetUnitStateRequest.Condition.FORCE, - upNodeState, newState); + UP_NODE_STATE, newState); assertTrue(result.settingWantedStateIsAllowed()); assertTrue(!result.wantedStateAlreadySet()); } @Test + public void testUnknownStorageNode() { + ContentCluster cluster = createCluster(createNodes(4)); + NodeStateChangeChecker nodeStateChangeChecker = new NodeStateChangeChecker( + 5 /* min storage nodes */, minRatioOfStorageNodesUp, requiredRedundancy, cluster.clusterInfo()); + NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( + new Node(NodeType.STORAGE, 10), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, + UP_NODE_STATE, MAINTENANCE_NODE_STATE); + assertFalse(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + assertThat(result.getReason(), is("Unknown node storage.10")); + } + + @Test public void testSafeSetStateDistributors() { NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(createCluster(createNodes(1))); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), containsString("Safe-set of node state is only supported for storage nodes")); @@ -146,7 +164,7 @@ public class NodeStateChangeCheckerTest { 5 /* min storage nodes */, minRatioOfStorageNodesUp, requiredRedundancy, cluster.clusterInfo()); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("There are only 4 storage nodes up, while config requires at least 5")); @@ -167,7 +185,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - maintenanceNodeState, upNodeState); + MAINTENANCE_NODE_STATE, UP_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); } @@ -187,7 +205,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SetUnitStateRequest.Condition.SAFE, - maintenanceNodeState, upNodeState); + MAINTENANCE_NODE_STATE, UP_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); } @@ -200,7 +218,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - new NodeState(NodeType.STORAGE, State.DOWN), upNodeState); + new NodeState(NodeType.STORAGE, State.DOWN), UP_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Refusing to set wanted state to up when it is currently in Down")); @@ -214,7 +232,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Distributor 0 says storage node 1 " + @@ -229,7 +247,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( new Node(NodeType.STORAGE, 3), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); } @@ -253,7 +271,7 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( new Node(NodeType.STORAGE, 1), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); } @@ -265,7 +283,7 @@ public class NodeStateChangeCheckerTest { cluster.clusterInfo().getStorageNodeInfo(1).setReportedState(new NodeState(NodeType.STORAGE, State.UP), 0); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( - nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, upNodeState, maintenanceNodeState); + nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Distributor node (0) has not reported any cluster state version yet.")); @@ -332,7 +350,7 @@ public class NodeStateChangeCheckerTest { } return nodeStateChangeChecker.evaluateTransition( - nodeStorage, clusterState, SetUnitStateRequest.Condition.SAFE, upNodeState, maintenanceNodeState); + nodeStorage, clusterState, SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); } private void setAllNodesUp(ContentCluster cluster, HostInfo distributorHostInfo) { @@ -402,11 +420,144 @@ public class NodeStateChangeCheckerTest { NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SetUnitStateRequest.Condition.SAFE, - upNodeState, maintenanceNodeState); + UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); } + @Test + public void testDownDisallowedByNonRetiredState() { + NodeStateChangeChecker.Result result = evaluateDownTransition( + defaultAllUpClusterState(), + State.UP, + currentClusterStateVersion, + 0); + assertFalse(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + assertEquals("Only retired nodes are allowed to be set to DOWN in safe mode - is Up", result.getReason()); + } + + @Test + public void testDownDisallowedByBuckets() { + NodeStateChangeChecker.Result result = evaluateDownTransition( + retiredClusterStateSuffix(), + State.UP, + currentClusterStateVersion, + 1); + assertFalse(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + assertEquals("The storage node manages 1 buckets", result.getReason()); + } + + @Test + public void testDownDisallowedByReportedState() { + NodeStateChangeChecker.Result result = evaluateDownTransition( + retiredClusterStateSuffix(), + State.INITIALIZING, + currentClusterStateVersion, + 0); + assertFalse(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + assertEquals("Reported state (Initializing) is not UP, so no bucket data is available", result.getReason()); + } + + @Test + public void testDownDisallowedByVersionMismatch() { + NodeStateChangeChecker.Result result = evaluateDownTransition( + retiredClusterStateSuffix(), + State.UP, + currentClusterStateVersion - 1, + 0); + assertFalse(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + assertEquals("Cluster controller at version 2 got info for storage node 1 at a different version 1", + result.getReason()); + } + + @Test + public void testAllowedToSetDown() { + NodeStateChangeChecker.Result result = evaluateDownTransition( + retiredClusterStateSuffix(), + State.UP, + currentClusterStateVersion, + 0); + assertTrue(result.settingWantedStateIsAllowed()); + assertFalse(result.wantedStateAlreadySet()); + } + + private NodeStateChangeChecker.Result evaluateDownTransition( + ClusterState clusterState, + State reportedState, + int hostInfoClusterStateVersion, + int lastAlldisksBuckets) { + ContentCluster cluster = createCluster(createNodes(4)); + NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); + + StorageNodeInfo nodeInfo = cluster.clusterInfo().getStorageNodeInfo(nodeStorage.getIndex()); + nodeInfo.setReportedState(new NodeState(NodeType.STORAGE, reportedState), 0); + nodeInfo.setHostInfo(createHostInfoWithMetrics(hostInfoClusterStateVersion, lastAlldisksBuckets)); + + return nodeStateChangeChecker.evaluateTransition( + nodeStorage, clusterState, SetUnitStateRequest.Condition.SAFE, + UP_NODE_STATE, DOWN_NODE_STATE); + } + + private ClusterState retiredClusterStateSuffix() { + return clusterState(String.format("version:%d distributor:4 storage:4 .%d.s:r", + currentClusterStateVersion, + nodeStorage.getIndex())); + } + + private static HostInfo createHostInfoWithMetrics(int clusterStateVersion, int lastAlldisksBuckets) { + return HostInfo.createHostInfo(String.format("{\n" + + " \"metrics\":\n" + + " {\n" + + " \"snapshot\":\n" + + " {\n" + + " \"from\":1494940706,\n" + + " \"to\":1494940766\n" + + " },\n" + + " \"values\":\n" + + " [\n" + + " {\n" + + " \"name\":\"vds.datastored.alldisks.buckets\",\n" + + " \"description\":\"buckets managed\",\n" + + " \"values\":\n" + + " {\n" + + " \"average\":262144.0,\n" + + " \"count\":1,\n" + + " \"rate\":0.016666,\n" + + " \"min\":262144,\n" + + " \"max\":262144,\n" + + " \"last\":%d\n" + + " },\n" + + " \"dimensions\":\n" + + " {\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\":\"vds.datastored.alldisks.docs\",\n" + + " \"description\":\"documents stored\",\n" + + " \"values\":\n" + + " {\n" + + " \"average\":154689587.0,\n" + + " \"count\":1,\n" + + " \"rate\":0.016666,\n" + + " \"min\":154689587,\n" + + " \"max\":154689587,\n" + + " \"last\":154689587\n" + + " },\n" + + " \"dimensions\":\n" + + " {\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"cluster-state-version\":%d\n" + + "}", + lastAlldisksBuckets, clusterStateVersion)); + } + private List<ConfiguredNode> createNodes(int count) { List<ConfiguredNode> nodes = new ArrayList<>(); for (int i = 0; i < count; i++) diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java index 3c4084d304c..9c928262321 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/WantedStateTest.java @@ -1,50 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.jrt.*; -import com.yahoo.jrt.StringValue; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; public class WantedStateTest extends FleetControllerTest { - private Supervisor supervisor; - - @Before - public void setUp() { - supervisor = new Supervisor(new Transport()); - } - - @After - public void tearDown() throws Exception { - if (supervisor != null) { - supervisor.transport().shutdown().join(); - supervisor = null; - } - super.tearDown(); - } - - public void setWantedState(DummyVdsNode node, State state, String reason) { - NodeState ns = new NodeState(node.getType(), state); - if (reason != null) ns.setDescription(reason); - Target connection = supervisor.connect(new Spec("localhost", fleetController.getRpcPort())); - Request req = new Request("setNodeState"); - req.parameters().add(new StringValue(node.getSlobrokName())); - req.parameters().add(new StringValue(ns.serialize())); - connection.invokeSync(req, timeoutS); - if (req.isError()) { - assertTrue("Failed to invoke setNodeState(): " + req.errorCode() + ": " + req.errorMessage(), false); - } - if (!req.checkReturnTypes("s")) { - assertTrue("Failed to invoke setNodeState(): Invalid return types.", false); - } - } - @Test public void testSettingStorageNodeMaintenanceAndBack() throws Exception { startingTest("WantedStateTest::testSettingStorageNodeMaintenanceAndBack()"); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java index d24b45817e0..fd93df4b2dc 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java @@ -32,7 +32,7 @@ public class HostInfoTest { HostInfo hostInfo = HostInfo.createHostInfo("{}"); assertThat(hostInfo.getVtag().getVersionOrNull(), is(nullValue())); assertThat(hostInfo.getDistributor().getStorageNodes().size(), is(0)); - assertThat(hostInfo.getMetrics().getValues().size(), is(0)); + assertThat(hostInfo.getMetrics().getMetrics().size(), is(0)); assertThat(hostInfo.getClusterStateVersionOrNull(), is(nullValue())); } @@ -51,7 +51,7 @@ public class HostInfoTest { assertThat(storageNodeList.get(0).getOpsLatenciesOrNull().getPut().getCount(), is(16L)); assertThat(storageNodeList.get(1).getOpsLatenciesOrNull().getPut().getCount(), is(18L)); assertThat(storageNodeList.get(0).getOpsLatenciesOrNull().getPut().getLatencyMsSum(), is(15L)); - List<Metrics.Metric> metrics = hostInfo.getMetrics().getValues(); + List<Metrics.Metric> metrics = hostInfo.getMetrics().getMetrics(); assertThat(metrics.size(), is(2)); Metrics.Value value = metrics.get(0).getValue(); assertThat(value.getLast(), is(5095L)); @@ -70,7 +70,7 @@ public class HostInfoTest { } HostInfo hostInfo = HostInfo.createHostInfo(json); // Check a value so not all code is removed by optimizer. - if (hostInfo.getMetrics().getValues().size() == -1) return; + if (hostInfo.getMetrics().getMetrics().size() == -1) return; } long end = System.currentTimeMillis(); System.out.println("Should take about 1.5 ms on fast machine, actually " + (end - start) / 10. + " ms."); diff --git a/component/src/main/java/com/yahoo/component/ComponentId.java b/component/src/main/java/com/yahoo/component/ComponentId.java index 50417c15b5e..687d0da617a 100644 --- a/component/src/main/java/com/yahoo/component/ComponentId.java +++ b/component/src/main/java/com/yahoo/component/ComponentId.java @@ -111,7 +111,8 @@ public final class ComponentId implements Comparable<ComponentId> { */ public String stringValue() { return stringValue; } - public @Override String toString() { + @Override + public String toString() { return spec.toString(); } @@ -126,7 +127,8 @@ public final class ComponentId implements Comparable<ComponentId> { return c.stringValue().equals(stringValue); } - public @Override int hashCode() { + @Override + public int hashCode() { return stringValue.hashCode(); } diff --git a/component/src/main/java/com/yahoo/component/ComponentSpecification.java b/component/src/main/java/com/yahoo/component/ComponentSpecification.java index 9662eee0f66..af8fbdcdbe1 100644 --- a/component/src/main/java/com/yahoo/component/ComponentSpecification.java +++ b/component/src/main/java/com/yahoo/component/ComponentSpecification.java @@ -91,7 +91,8 @@ public final class ComponentSpecification { */ public String stringValue() { return stringValue; } - public @Override String toString() { + @Override + public String toString() { return toId().toString(); } @@ -102,7 +103,8 @@ public final class ComponentSpecification { return c.stringValue.equals(this.stringValue()); } - public @Override int hashCode() { + @Override + public int hashCode() { return stringValue.hashCode(); } diff --git a/component/src/main/java/com/yahoo/component/Spec.java b/component/src/main/java/com/yahoo/component/Spec.java index a3259484b4a..54cf52a3887 100644 --- a/component/src/main/java/com/yahoo/component/Spec.java +++ b/component/src/main/java/com/yahoo/component/Spec.java @@ -51,7 +51,8 @@ final class Spec<VERSION> { } } - public @Override String toString() { + @Override + public String toString() { if (isNonEmpty(version) || (namespace != null)) { StringBuilder builder = new StringBuilder(name); if (isNonEmpty(version)) { diff --git a/component/src/main/java/com/yahoo/component/Version.java b/component/src/main/java/com/yahoo/component/Version.java index 8ef540c2ae7..53931406bb0 100644 --- a/component/src/main/java/com/yahoo/component/Version.java +++ b/component/src/main/java/com/yahoo/component/Version.java @@ -360,10 +360,20 @@ public final class Version implements Comparable<Version> { return getQualifier().compareTo(other.getQualifier()); } + + /** Returns whether this version number (ignoring qualifier) is strictly lower than the given version */ + public boolean isBefore(Version other) { + if (this.major == other.major && this.minor == other.minor) return this.micro < other.micro; + if (this.major == other.major) return this.minor < other.minor; + return this.major < other.major; + } - /** - * Creates a version specification that only matches this version. - */ + /** Returns whether this version number (ignoring qualifier) is strictly higher than the given version */ + public boolean isAfter(Version other) { + return ! this.isBefore(other) && ! this.equals(other); + } + + /** Creates a version specification that only matches this version */ public VersionSpecification toSpecification() { if (this == emptyVersion) return VersionSpecification.emptyVersionSpecification; diff --git a/component/src/main/java/com/yahoo/component/provider/FreezableClass.java b/component/src/main/java/com/yahoo/component/provider/FreezableClass.java index c94c8a0bbac..895f642e315 100644 --- a/component/src/main/java/com/yahoo/component/provider/FreezableClass.java +++ b/component/src/main/java/com/yahoo/component/provider/FreezableClass.java @@ -30,7 +30,8 @@ public class FreezableClass implements Freezable { } /** Clones this. The clone is <i>not</i> frozen */ - public @Override FreezableClass clone() { + @Override + public FreezableClass clone() { try { FreezableClass clone=(FreezableClass)super.clone(); clone.frozen = false; diff --git a/component/src/main/java/com/yahoo/component/provider/FreezableComponent.java b/component/src/main/java/com/yahoo/component/provider/FreezableComponent.java index 9d51b0b5819..d2189c3a48b 100644 --- a/component/src/main/java/com/yahoo/component/provider/FreezableComponent.java +++ b/component/src/main/java/com/yahoo/component/provider/FreezableComponent.java @@ -38,7 +38,8 @@ public class FreezableComponent extends AbstractComponent implements Freezable { } /** Clones this. The clone will <i>not</i> be frozen */ - public @Override FreezableComponent clone() { + @Override + public FreezableComponent clone() { FreezableComponent clone=(FreezableComponent)super.clone(); clone.frozen = false; return clone; diff --git a/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java b/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java index d1d82b2de79..7bb2eae10a5 100644 --- a/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java +++ b/component/src/main/java/com/yahoo/component/provider/ListenableFreezableClass.java @@ -34,7 +34,8 @@ public class ListenableFreezableClass extends FreezableClass implements Listenab } /** Clones this. The clone is <i>not</i> frozen and has no listeners. */ - public @Override ListenableFreezableClass clone() { + @Override + public ListenableFreezableClass clone() { ListenableFreezableClass clone=(ListenableFreezableClass)super.clone(); clone.executionList = new ExecutionList(); return clone; diff --git a/component/src/test/java/com/yahoo/component/VersionTestCase.java b/component/src/test/java/com/yahoo/component/VersionTestCase.java index 99a27347d5f..cca1fd3be3f 100644 --- a/component/src/test/java/com/yahoo/component/VersionTestCase.java +++ b/component/src/test/java/com/yahoo/component/VersionTestCase.java @@ -3,12 +3,18 @@ package com.yahoo.component; import com.yahoo.text.Utf8Array; import com.yahoo.text.Utf8String; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author bratseth */ -public class VersionTestCase extends junit.framework.TestCase { +public class VersionTestCase { + @Test public void testPrimitiveCreation() { Version version=new Version(1,2,3,"qualifier"); assertEquals(1,version.getMajor()); @@ -17,6 +23,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("qualifier",version.getQualifier()); } + @Test public void testUnderspecifiedPrimitiveCreation() { Version version=new Version(1); assertEquals(1,version.getMajor()); @@ -26,6 +33,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("",version.getQualifier()); } + @Test public void testStringCreation() { Version version=new Version("1.2.3.qualifier"); assertEquals(1,version.getMajor()); @@ -33,6 +41,8 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals(3,version.getMicro()); assertEquals("qualifier",version.getQualifier()); } + + @Test public void testUtf8StringCreation() { Version version=new Version((Utf8Array)new Utf8String("1.2.3.qualifier")); assertEquals(1,version.getMajor()); @@ -41,6 +51,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("qualifier",version.getQualifier()); } + @Test public void testUnderspecifiedStringCreation() { Version version=new Version("1"); assertEquals(1,version.getMajor()); @@ -49,6 +60,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("",version.getQualifier()); } + @Test public void testEquality() { assertEquals(new Version(),Version.emptyVersion); assertEquals(new Version(),new Version("")); @@ -59,6 +71,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals(new Version(1,2,3,"qualifier"),new Version("1.2.3.qualifier")); } + @Test public void testToString() { assertEquals("",new Version().toString()); assertEquals("1",new Version(1).toString()); @@ -67,6 +80,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("1.2.3.qualifier",new Version(1,2,3,"qualifier").toString()); } + @Test public void testToFullString() { assertEquals("0.0.0",new Version().toFullString()); assertEquals("1.0.0",new Version(1).toFullString()); @@ -75,6 +89,7 @@ public class VersionTestCase extends junit.framework.TestCase { assertEquals("1.2.3.qualifier",new Version(1,2,3,"qualifier").toFullString()); } + @Test public void testOrder() { assertTrue(new Version("1.2.3").compareTo(new Version("1.2.3"))==0); assertTrue(new Version("1.2.3").compareTo(new Version("1.2.4"))<0); @@ -85,5 +100,27 @@ public class VersionTestCase extends junit.framework.TestCase { assertTrue(new Version("1.0.0").compareTo(new Version("1"))==0); } + + @Test + public void testIsBefore() { + assertFalse(new Version("1.2.3").isBefore(new Version("0.2.3"))); + assertFalse(new Version("1.2.3").isBefore(new Version("1.1.3"))); + assertFalse(new Version("1.2.3").isBefore(new Version("1.2.2"))); + assertFalse(new Version("1.2.3").isBefore(new Version("1.2.3"))); + assertTrue( new Version("1.2.3").isBefore(new Version("1.2.4"))); + assertTrue( new Version("1.2.3").isBefore(new Version("1.3.3"))); + assertTrue( new Version("1.2.3").isBefore(new Version("2.2.3"))); + } + + @Test + public void testIsAfter() { + assertTrue( new Version("1.2.3").isAfter(new Version("0.2.3"))); + assertTrue( new Version("1.2.3").isAfter(new Version("1.1.3"))); + assertTrue( new Version("1.2.3").isAfter(new Version("1.2.2"))); + assertFalse(new Version("1.2.3").isAfter(new Version("1.2.3"))); + assertFalse(new Version("1.2.3").isAfter(new Version("1.2.4"))); + assertFalse(new Version("1.2.3").isAfter(new Version("1.3.3"))); + assertFalse(new Version("1.2.3").isAfter(new Version("2.2.3"))); + } } diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 6bb1191ff8f..3acfd8bb823 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -126,6 +126,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> + <skip>false</skip> <compilerArgs> <arg>-Xlint:all</arg> <arg>-Werror</arg> @@ -140,6 +141,10 @@ <goals> <goal>test-jar</goal> </goals> + <configuration> + <!-- ugly workaround for https://issues.apache.org/jira/browse/MJAR-138 --> + <skip>false</skip> + </configuration> </execution> </executions> </plugin> diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 8fa2d482cab..9e057edad15 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -75,6 +75,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> + <skip>false</skip> <compilerArgs> <arg>-Xlint:all</arg> <arg>-Werror</arg> @@ -95,6 +96,10 @@ <goals> <goal>test-jar</goal> </goals> + <configuration> + <!-- ugly workaround for https://issues.apache.org/jira/browse/MJAR-138 --> + <skip>false</skip> + </configuration> </execution> </executions> </plugin> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index d54437772e6..7535d39e035 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -4,34 +4,111 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.io.IOUtils; import com.yahoo.text.XML; import org.w3c.dom.Element; import java.io.BufferedReader; import java.io.FileReader; +import java.io.IOException; import java.io.Reader; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Specifies the environments and regions to which an application should be deployed. * This may be used both for inspection as part of an application model and to answer * queries about deployment from the command line. A main method is included for the latter usage. + * + * A deployment consists of a number of steps executed in the order listed in deployment xml, as + * well as some additional settings. + * + * This is immutable. * * @author bratseth */ public class DeploymentSpec { + /** The empty deployment spec, specifying no zones or rotation, and defaults for all settings */ + public static final DeploymentSpec empty = new DeploymentSpec(Optional.empty(), + UpgradePolicy.defaultPolicy, + ImmutableList.of(), + "<deployment version='1.0'/>"); + private final Optional<String> globalServiceId; private final UpgradePolicy upgradePolicy; - private final List<DeclaredZone> zones; + private final List<Step> steps; + private final String xmlForm; + + public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, List<Step> steps) { + this(globalServiceId, upgradePolicy, steps, null); + } - public DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, List<DeclaredZone> zones) { + private DeploymentSpec(Optional<String> globalServiceId, UpgradePolicy upgradePolicy, + List<Step> steps, String xmlForm) { + validateTotalDelay(steps); this.globalServiceId = globalServiceId; this.upgradePolicy = upgradePolicy; - this.zones = ImmutableList.copyOf(zones); + this.steps = ImmutableList.copyOf(completeSteps(new ArrayList<>(steps))); + this.xmlForm = xmlForm; + } + + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ + private static void validateTotalDelay(List<Step> steps) { + long totalDelaySeconds = steps.stream().filter(step -> step instanceof Delay) + .mapToLong(delay -> ((Delay)delay).duration().getSeconds()) + .sum(); + if (totalDelaySeconds > Duration.ofHours(24).getSeconds()) + throw new IllegalArgumentException("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) + + " but max 24 hours is allowed"); + } + + /** Adds missing required steps and reorders steps to a permissible order */ + private static List<Step> completeSteps(List<Step> steps) { + // Ensure no duplicate deployments to the same zone + steps = new ArrayList<>(new LinkedHashSet<>(steps)); + + // Add staging if required and missing + if (steps.stream().anyMatch(step -> step.deploysTo(Environment.prod)) && + steps.stream().noneMatch(step -> step.deploysTo(Environment.staging))) { + steps.add(new DeclaredZone(Environment.staging)); + } + + // Add test if required and missing + if (steps.stream().anyMatch(step -> step.deploysTo(Environment.staging)) && + steps.stream().noneMatch(step -> step.deploysTo(Environment.test))) { + steps.add(new DeclaredZone(Environment.test)); + } + + // Enforce order test, staging, prod + DeclaredZone testStep = remove(Environment.test, steps); + if (testStep != null) + steps.add(0, testStep); + DeclaredZone stagingStep = remove(Environment.staging, steps); + if (stagingStep != null) + steps.add(1, stagingStep); + + return steps; + } + + /** + * Removes the first occurrence of a deployment step to the given environment and returns it. + * + * @param environment + * @return the removed step, or null if it is not present + */ + private static DeclaredZone remove(Environment environment, List<Step> steps) { + for (int i = 0; i < steps.size(); i++) { + if (steps.get(i).deploysTo(environment)) + return (DeclaredZone)steps.remove(i); + } + return null; } /** Returns the ID of the service to expose through global routing, if present */ @@ -42,13 +119,22 @@ public class DeploymentSpec { /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ public UpgradePolicy upgradePolicy() { return upgradePolicy; } - /** Returns the zones this declares as a read-only list. */ - public List<DeclaredZone> zones() { return zones; } + /** Returns the deployment steps of this in the order they will be performed */ + public List<Step> steps() { return steps; } + + /** Returns only the DeclaredZone deployment steps of this in the order they will be performed */ + public List<DeclaredZone> zones() { + return steps.stream().filter(step -> step instanceof DeclaredZone).map(DeclaredZone.class::cast) + .collect(Collectors.toList()); + } + + /** Returns the XML form of this spec, or null if it was not created by fromXml or is the empty spec */ + public String xmlForm() { return xmlForm; } /** Returns whether this deployment spec specifies the given zone, either implicitly or explicitly */ public boolean includes(Environment environment, Optional<RegionName> region) { - for (DeclaredZone declaredZone : zones) - if (declaredZone.matches(environment, region)) return true; + for (Step step : steps) + if (step.deploysTo(environment, region)) return true; return false; } @@ -58,31 +144,63 @@ public class DeploymentSpec { * @throws IllegalArgumentException if the XML is invalid */ public static DeploymentSpec fromXml(Reader reader) { - List<DeclaredZone> zones = new ArrayList<>(); - Element root = XML.getDocument(reader).getDocumentElement(); + try { + return fromXml(IOUtils.readAll(reader)); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read deployment spec", e); + } + } + + /** + * Creates a deployment spec from XML. + * + * @throws IllegalArgumentException if the XML is invalid + */ + public static DeploymentSpec fromXml(String xmlForm) { + List<Step> steps = new ArrayList<>(); Optional<String> globalServiceId = Optional.empty(); + Element root = XML.getDocument(xmlForm).getDocumentElement(); for (Element environmentTag : XML.getChildren(root)) { if ( ! isEnvironmentName(environmentTag.getTagName())) continue; + Environment environment = Environment.from(environmentTag.getTagName()); - List<Element> regionTags = XML.getChildren(environmentTag, "region"); - if (regionTags.isEmpty()) { - zones.add(new DeclaredZone(environment, Optional.empty(), false)); + + if (environment == Environment.prod) { + for (Element stepTag : XML.getChildren(environmentTag)) { + if (stepTag.getTagName().equals("delay")) + steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + + longAttribute("minutes", stepTag) * 60 + + longAttribute("seconds", stepTag)))); + else // a region: deploy step + steps.add(new DeclaredZone(environment, + Optional.of(RegionName.from(XML.getValue(stepTag).trim())), + readActive(stepTag))); + } } else { - for (Element regionTag : regionTags) { - RegionName region = RegionName.from(XML.getValue(regionTag).trim()); - boolean active = environment == Environment.prod && readActive(regionTag); - zones.add(new DeclaredZone(environment, Optional.of(region), active)); - } + steps.add(new DeclaredZone(environment)); } - if (Environment.prod.equals(environment)) { + if (environment == Environment.prod) globalServiceId = readGlobalServiceId(environmentTag); - } else if (readGlobalServiceId(environmentTag).isPresent()) { + else if (readGlobalServiceId(environmentTag).isPresent()) throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag."); - } } - return new DeploymentSpec(globalServiceId, readUpgradePolicy(root), zones); + return new DeploymentSpec(globalServiceId, readUpgradePolicy(root), steps, xmlForm); + } + + /** Returns the given attribute as an integer, or 0 if it is not present */ + private static long longAttribute(String attributeName, Element tag) { + String value = tag.getAttribute(attributeName); + if (value == null || value.isEmpty()) return 0; + try { + return Long.parseLong(value); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected an integer for attribute '" + attributeName + + "' but got '" + value + "'"); + } } private static boolean isEnvironmentName(String tagName) { @@ -139,7 +257,7 @@ public class DeploymentSpec { } /** This may be invoked by a continuous build */ - public static void main (String[] args) { + public static void main(String[] args) { if (args.length != 2 && args.length != 3) { System.err.println("Usage: DeploymentSpec [file] [environment] [region]?" + "Returns 0 if the specified zone matches the deployment spec, 1 otherwise"); @@ -161,7 +279,37 @@ public class DeploymentSpec { } } - public static class DeclaredZone { + /** A delpoyment step */ + public abstract static class Step { + + /** Returns whether this step deploys to the given region */ + public final boolean deploysTo(Environment environment) { + return deploysTo(environment, Optional.empty()); + } + + /** Returns whether this step deploys to the given environment, and (if specified) region */ + public abstract boolean deploysTo(Environment environment, Optional<RegionName> region); + + } + + /** A deployment step which is to wait for some time before progressing to the next step */ + public static class Delay extends Step { + + private final Duration duration; + + public Delay(Duration duration) { + this.duration = duration; + } + + public Duration duration() { return duration; } + + @Override + public boolean deploysTo(Environment environment, Optional<RegionName> region) { return false; } + + } + + /** A deployment step which is to run deployment in a particular zone */ + public static class DeclaredZone extends Step { private final Environment environment; @@ -169,7 +317,15 @@ public class DeploymentSpec { private final boolean active; + public DeclaredZone(Environment environment) { + this(environment, Optional.empty(), false); + } + public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active) { + if (environment != Environment.prod && region.isPresent()) + throw new IllegalArgumentException("Non-prod environments cannot specify a region"); + if (environment == Environment.prod && ! region.isPresent()) + throw new IllegalArgumentException("Prod environments must be specified with a region"); this.environment = environment; this.region = region; this.active = active; @@ -183,23 +339,31 @@ public class DeploymentSpec { /** Returns whether this zone should receive production traffic */ public boolean active() { return active; } + @Override + public boolean deploysTo(Environment environment, Optional<RegionName> region) { + if (environment != this.environment) return false; + if (region.isPresent() && ! region.equals(this.region)) return false; + return true; + } + + // TODO: Remove when no version older than 6.111 is deployed anywhere public boolean matches(Environment environment, Optional<RegionName> region) { - if (environment.equals(this.environment) && region.equals(this.region)) return true; - if ( ! region.isPresent() && prerequisite(environment)) return true; - return false; - } - - /** - * Returns whether deployment in the given environment is a prerequisite of deployment in this environment - * - * The required progression leading to prerequisites is test, staging, prod. - */ - private boolean prerequisite(Environment environment) { - if (this.environment == Environment.prod) - return environment == Environment.staging || environment == Environment.test; - if (this.environment == Environment.staging) - return environment == Environment.test; - return false; + return deploysTo(environment, region); + } + + @Override + public int hashCode() { + return Objects.hash(environment, region); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof DeclaredZone)) return false; + DeclaredZone other = (DeclaredZone)o; + if (this.environment != other.environment) return false; + if ( ! this.region.equals(other.region())) return false; + return true; } } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index f1a9840bd88..8de8a9c2da4 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -19,15 +19,15 @@ public class DeploymentSpecTest { @Test public void testSpec() { - StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <test/>" + - "</deployment>" - ); + String specXml = "<deployment version='1.0'>" + + " <test/>" + + "</deployment>"; + StringReader r = new StringReader(specXml); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(1, spec.zones().size()); - assertEquals(Environment.test, spec.zones().get(0).environment()); + assertEquals(specXml, spec.xmlForm()); + assertEquals(1, spec.steps().size()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); assertTrue(spec.includes(Environment.test, Optional.empty())); assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); assertFalse(spec.includes(Environment.staging, Optional.empty())); @@ -44,8 +44,9 @@ public class DeploymentSpecTest { ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(1, spec.zones().size()); - assertEquals(Environment.staging, spec.zones().get(0).environment()); + assertEquals(2, spec.steps().size()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); assertTrue(spec.includes(Environment.test, Optional.empty())); assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); assertTrue(spec.includes(Environment.staging, Optional.empty())); @@ -65,15 +66,17 @@ public class DeploymentSpecTest { ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(2, spec.zones().size()); + assertEquals(4, spec.steps().size()); + + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); - assertEquals(Environment.prod, spec.zones().get(0).environment()); - assertEquals("us-east1", spec.zones().get(0).region().get().value()); - assertFalse(spec.zones().get(0).active()); + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); - assertEquals(Environment.prod, spec.zones().get(1).environment()); - assertEquals("us-west1", spec.zones().get(1).region().get().value()); - assertTrue(spec.zones().get(1).active()); + assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); + + assertTrue(spec.steps().get(3).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(3)).active()); assertTrue(spec.includes(Environment.test, Optional.empty())); assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); @@ -91,28 +94,33 @@ public class DeploymentSpecTest { StringReader r = new StringReader( "<deployment version='1.0'>" + " <test/>" + + " <test/>" + " <staging/>" + " <prod>" + " <region active='false'>us-east1</region>" + + " <region active='false'>us-east1</region>" + + " <delay hours='3' minutes='30'/>" + " <region active='true'>us-west1</region>" + " </prod>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(5, spec.steps().size()); assertEquals(4, spec.zones().size()); - assertEquals(Environment.test, spec.zones().get(0).environment()); + assertTrue(spec.steps().get(0).deploysTo(Environment.test)); + + assertTrue(spec.steps().get(1).deploysTo(Environment.staging)); - assertEquals(Environment.staging, spec.zones().get(1).environment()); + assertTrue(spec.steps().get(2).deploysTo(Environment.prod, Optional.of(RegionName.from("us-east1")))); + assertFalse(((DeploymentSpec.DeclaredZone)spec.steps().get(2)).active()); - assertEquals(Environment.prod, spec.zones().get(2).environment()); - assertEquals("us-east1", spec.zones().get(2).region().get().value()); - assertFalse(spec.zones().get(2).active()); + assertTrue(spec.steps().get(3) instanceof DeploymentSpec.Delay); + assertEquals(3 * 60 * 60 + 30 * 60, ((DeploymentSpec.Delay)spec.steps().get(3)).duration().getSeconds()); - assertEquals(Environment.prod, spec.zones().get(3).environment()); - assertEquals("us-west1", spec.zones().get(3).region().get().value()); - assertTrue(spec.zones().get(3).active()); + assertTrue(spec.steps().get(4).deploysTo(Environment.prod, Optional.of(RegionName.from("us-west1")))); + assertTrue(((DeploymentSpec.DeclaredZone)spec.steps().get(4)).active()); assertTrue(spec.includes(Environment.test, Optional.empty())); assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); @@ -193,4 +201,36 @@ public class DeploymentSpecTest { assertEquals("canary", spec.upgradePolicy().toString()); } + @Test + public void maxDelayExceeded() { + try { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <delay hours='23'/>" + + " <region active='true'>us-central-1</region>" + + " <delay minutes='59' seconds='61'/>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + DeploymentSpec.fromXml(r); + fail("Expected exception due to exceeding the max total delay"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("The total delay specified is PT24H1S but max 24 hours is allowed", e.getMessage()); + } + } + + @Test + public void testEmpty() { + assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); + assertTrue(DeploymentSpec.empty.steps().isEmpty()); + assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java index 442673e3a46..37f18e44b91 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java @@ -42,7 +42,8 @@ public class Ranking implements Cloneable, Serializable { public void setNormal(boolean n) { this.normal = n; } /** Returns true if the given rank settings are the same */ - public @Override boolean equals(Object o) { + @Override + public boolean equals(Object o) { if ( ! (o instanceof Ranking)) return false; Ranking other=(Ranking)o; @@ -52,15 +53,18 @@ public class Ranking implements Cloneable, Serializable { return true; } - public @Override int hashCode() { + @Override + public int hashCode() { return java.util.Objects.hash(filter, literal, normal); } - public @Override String toString() { + @Override + public String toString() { return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]"; } - public @Override Ranking clone() { + @Override + public Ranking clone() { try { return (Ranking)super.clone(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java index 8c238c104fe..34db554773c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java @@ -48,7 +48,8 @@ public final class Sorting implements Cloneable, Serializable { (strength == other.strength); } - public @Override Sorting clone() { + @Override + public Sorting clone() { try { return (Sorting)super.clone(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 3eadd381508..b9d7a5953ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -74,10 +74,12 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } public String getName() { return name; } public String getValue() { return value; } - public @Override int hashCode() { + @Override + public int hashCode() { return name.hashCode() + 17*value.hashCode(); } - public @Override boolean equals(Object obj) { + @Override + public boolean equals(Object obj) { if (!(obj instanceof Property)) { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java new file mode 100644 index 00000000000..af9740a3633 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java @@ -0,0 +1,34 @@ +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerClusterVerifier; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.util.Collections; +import java.util.Set; + +/** + * @author baldersheim + * Verifies that all containers added are ClusterControllerContainers and that filters away Linguistics components. + */ +public class ClusterControllerClusterVerifier implements ContainerClusterVerifier { + static final Set<ComponentSpecification> unwantedComponents = Collections.singleton(new SimpleComponent(ContainerCluster.SIMPLE_LINGUISTICS_PROVIDER).getClassId()); + @Override + public boolean acceptComponent(Component component) { + return ! unwantedComponents.contains(component.getClassId()); + } + + @Override + public boolean acceptContainer(Container container) { + return container instanceof ClusterControllerContainer; + } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + builder.maxthreads(10); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 9a573172ec9..2db5c8ae927 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -28,8 +28,7 @@ import java.util.TreeSet; public class ClusterControllerContainer extends Container implements BundlesConfig.Producer, ZookeeperServerConfig.Producer, - QrStartConfig.Producer, - ThreadpoolConfig.Producer + QrStartConfig.Producer { private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); private static final ComponentSpecification ZKFACADE_BUNDLE = new ComponentSpecification("zkfacade"); @@ -115,10 +114,6 @@ public class ClusterControllerContainer extends Container implements public void getConfig(QrStartConfig.Builder builder) { builder.jvm(new QrStartConfig.Jvm.Builder().heapsize(512)); } - @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - builder.maxthreads(10); - } int getIndex() { return index; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java index ab2d89023da..c0ea2822e46 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetricsConsumer.java @@ -1,21 +1,29 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.google.common.collect.ImmutableList; + +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; +import static java.util.Collections.emptyList; + /** - * This class sets up the default 'vespa' metrics consumer. + * This class sets up the default 'Vespa' metrics consumer. * - * @author <a href="mailto:trygve@yahoo-inc.com">Trygve Bolsø Berdal</a> + * @author trygve * @author gjoranv */ public class DefaultMetricsConsumer { public static final String VESPA_CONSUMER_ID = "Vespa"; - private static final MetricSet vespaMetricSet = new VespaMetricSet(); + private static final MetricSet defaultConsumerMetrics = new MetricSet("default-consumer", + emptyList(), + ImmutableList.of(vespaMetricSet, systemMetricSet)); @SuppressWarnings("UnusedDeclaration") public static MetricsConsumer getDefaultMetricsConsumer() { - return new MetricsConsumer(VESPA_CONSUMER_ID, vespaMetricSet); + return new MetricsConsumer(VESPA_CONSUMER_ID, defaultConsumerMetrics); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java new file mode 100644 index 00000000000..68346477f23 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java @@ -0,0 +1,32 @@ +package com.yahoo.vespa.model.admin.monitoring; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Encapsulates a minimal set of Vespa metrics to be used as default for all metrics consumers. + * + * @author leandroalves + */ +public class DefaultVespaMetrics { + public static final MetricSet defaultVespaMetricSet = createDefaultVespaMetricSet(); + + private static MetricSet createDefaultVespaMetricSet() { + + Set<Metric> defaultContainerMetrics = + ImmutableSet.of(new Metric("feed.operations.rate") + ); + + Set<Metric> defaultContentMetrics = + ImmutableSet.of(new Metric("content.proton.resource_usage.feeding_blocked.last") + ); + + Set<Metric> defaultMetrics = ImmutableSet.<Metric>builder() + .addAll(defaultContainerMetrics) + .addAll(defaultContentMetrics) + .build(); + + return new MetricSet("default-vespa", defaultMetrics); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java new file mode 100644 index 00000000000..5967771d904 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.monitoring; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * @author gjoranv + */ +@SuppressWarnings("UnusedDeclaration") // Used by model amenders +public class SystemMetrics { + public static final String CPU_UTIL = "cpu.util"; + public static final String DISK_LIMIT = "disk.limit"; + public static final String DISK_USED = "disk.used"; + public static final String DISK_UTIL = "disk.util"; + public static final String MEM_LIMIT = "mem.limit"; + public static final String MEM_USED = "mem.used"; + public static final String MEM_UTIL = "mem.util"; + + public static final MetricSet systemMetricSet = createSystemMetricSet(); + + private static MetricSet createSystemMetricSet() { + Set<Metric> dockerNodeMetrics = + ImmutableSet.of(new Metric(CPU_UTIL), + new Metric(DISK_LIMIT), + new Metric(DISK_USED), + new Metric(DISK_UTIL), + new Metric(MEM_LIMIT), + new Metric(MEM_USED), + new Metric(MEM_UTIL) + ); + + Set<Metric> nonDockerNodeMetrics = + // Disk metrics should be based on /home, or else '/' - or simply add filesystem as dimension + ImmutableSet.of(new Metric("cpu.busy.pct", CPU_UTIL), + new Metric("mem.used.pct", MEM_UTIL), + new Metric("mem.active.kb", MEM_USED), + new Metric("mem.total.kb", MEM_LIMIT), + new Metric("used.kb", DISK_USED) + ); + + Set<Metric> systemMetrics = ImmutableSet.<Metric>builder() + .addAll(dockerNodeMetrics) + .addAll(nonDockerNodeMetrics) + .build(); + + return new MetricSet("system", systemMetrics); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 15275df0b87..21c6f5a68f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -2,24 +2,26 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static java.util.Collections.singleton; + /** + * Encapsulates vespa service metrics. + * * @author gjoranv */ @SuppressWarnings("UnusedDeclaration") // Used by model amenders -public class VespaMetricSet extends MetricSet { - - private static final Set<Metric> vespaMetrics = getVespaMetrics(); +public class VespaMetricSet { - public VespaMetricSet() { - super("vespa", vespaMetrics, Collections.emptySet()); - } + public static final MetricSet vespaMetricSet = new MetricSet("vespa", + getVespaMetrics(), + singleton(defaultVespaMetricSet)); private static Set<Metric> getVespaMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); metrics.addAll(getSearchNodeMetrics()); metrics.addAll(getStorageMetrics()); @@ -28,10 +30,17 @@ public class VespaMetricSet extends MetricSet { metrics.addAll(getQrserverMetrics()); metrics.addAll(getContainerMetrics()); metrics.addAll(getConfigServerMetrics()); + metrics.addAll(getOtherMetrics()); return Collections.unmodifiableSet(metrics); } + private static Set<Metric> getOtherMetrics() { + Set<Metric> metrics = new LinkedHashSet<>(); + metrics.add(new Metric("slobrok.heartbeats.failed.count", "slobrok.heartbeats.failed")); + return metrics; + } + private static Set<Metric> getConfigServerMetrics() { Set<Metric> metrics =new LinkedHashSet<>(); @@ -48,7 +57,7 @@ public class VespaMetricSet extends MetricSet { } private static Set<Metric> getContainerMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("serverRejectedRequests.rate")); metrics.add(new Metric("serverRejectedRequests.count")); @@ -105,7 +114,7 @@ public class VespaMetricSet extends MetricSet { } private static Set<Metric> getDocprocMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); // per chain metrics.add(new Metric("documents_processed.rate", "documents_processed")); @@ -116,9 +125,10 @@ public class VespaMetricSet extends MetricSet { private static Set<Metric> getQrserverMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("peak_qps.average", "peak_qps")); + metrics.add(new Metric("peak_qps.max", "peak_qps")); metrics.add(new Metric("search_connections.average", "search_connections")); metrics.add(new Metric("active_queries.average", "active_queries")); + metrics.add(new Metric("feed.latency.average")); metrics.add(new Metric("queries.rate", "queries")); metrics.add(new Metric("query_latency.average", "mean_query_latency")); metrics.add(new Metric("query_latency.max", "max_query_latency")); @@ -154,7 +164,7 @@ public class VespaMetricSet extends MetricSet { } private static Set<Metric> getSearchNodeMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("proton.numstoreddocs.last", "documents_total")); metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready")); @@ -193,7 +203,6 @@ public class VespaMetricSet extends MetricSet { metrics.add(new Metric("content.proton.resource_usage.memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); - metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.last")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); @@ -245,7 +254,7 @@ public class VespaMetricSet extends MetricSet { } private static Set<Metric> getStorageMetrics() { - Set<Metric> metrics =new LinkedHashSet<>(); + Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs")); metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java index bbeccb52bcb..c8f6d787688 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java @@ -2,12 +2,15 @@ package com.yahoo.vespa.model.admin.monitoring.builder; import com.yahoo.vespa.model.admin.monitoring.MetricSet; -import com.yahoo.vespa.model.admin.monitoring.VespaMetricSet; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; + /** * A data object for predefined metric sets. * @@ -16,7 +19,9 @@ import java.util.Map; public class PredefinedMetricSets { public static final Map<String, MetricSet> predefinedMetricSets = toMapById( - new VespaMetricSet() + defaultVespaMetricSet, + vespaMetricSet, + systemMetricSet ); private static Map<String, MetricSet> toMapById(MetricSet... metricSets) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index 130e4555e19..ad08ddd08a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring.builder.xml; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.text.XML; +import com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics; import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; @@ -15,6 +16,8 @@ import java.util.Map; import java.util.stream.Collectors; import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; /** * @author gjoranv @@ -52,6 +55,9 @@ public class MetricsBuilder { .map(metricSetElement -> availableMetricSets.get(metricSetElement.getAttribute(ID_ATTRIBUTE))) .collect(Collectors.toCollection(LinkedList::new)); + metricSets.add(defaultVespaMetricSet); + metricSets.add(systemMetricSet); + return new MetricSet(metricSetId(consumerId), metrics, metricSets); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 851f8ce476b..c7f4fa0b1bd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -6,10 +6,14 @@ import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.model.*; -import com.yahoo.vespa.model.admin.*; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.Configserver; +import com.yahoo.vespa.model.admin.Logserver; +import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerClusterVerifier; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; @@ -17,8 +21,8 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; -import java.util.ArrayList; import java.util.List; +import java.util.ArrayList; import java.util.logging.Level; /** @@ -81,7 +85,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { if (standaloneZooKeeper) { parent = new ClusterControllerCluster(parent, "standalone"); } - ContainerCluster cluster = new ContainerCluster(parent, "cluster-controllers", "cluster-controllers"); + ContainerCluster cluster = new ContainerCluster(parent, "cluster-controllers", "cluster-controllers", new ClusterControllerClusterVerifier()); ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster); List<Container> containers = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6015aaaaa20..dbae6851977 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -24,6 +24,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -133,7 +134,9 @@ public final class ContainerCluster ClusterInfoConfig.Producer, ServletPathsConfig.Producer, RoutingProviderConfig.Producer, - ConfigserverConfig.Producer { + ConfigserverConfig.Producer, + ThreadpoolConfig.Producer +{ /** * URI prefix used for internal, usually programmatic, APIs. URIs using this @@ -147,6 +150,7 @@ public final class ContainerCluster public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; public static final String STATISTICS_HANDLER_CLASS = "com.yahoo.container.config.StatisticsRequestHandler"; + public static final String SIMPLE_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.SimpleLinguisticsProvider"; public static final String ROOT_HANDLER_BINDING = "*://*/"; @@ -170,6 +174,7 @@ public final class ContainerCluster protected final ComponentGroup<Component<?, ?>> componentGroup; private final ConfigProducerGroup<RestApi> restApiGroup; private final ConfigProducerGroup<Servlet> servletGroup; + private final ContainerClusterVerifier clusterVerifier; private Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); private MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory; @@ -182,8 +187,26 @@ public final class ContainerCluster private Optional<String> hostClusterId = Optional.empty(); private Optional<Integer> memoryPercentage = Optional.empty(); + private static class AcceptAllVerifier implements ContainerClusterVerifier { + @Override + public boolean acceptComponent(Component component) { return true; } + + @Override + public boolean acceptContainer(Container container) { return true; } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + + } + } + public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name) { + this(parent, subId, name, new AcceptAllVerifier()); + } + + public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, ContainerClusterVerifier verifier) { super(parent, subId); + this.clusterVerifier = verifier; this.name = name; this.zone = getRoot() != null ? getRoot().getDeployState().zone() : Zone.defaultZone(); componentGroup = new ComponentGroup<>(this, "component"); @@ -195,7 +218,7 @@ public final class ContainerCluster // TODO better modelling addSimpleComponent(ThreadPoolProvider.class); addSimpleComponent("com.yahoo.jdisc.http.filter.SecurityFilterInvoker"); - addSimpleComponent("com.yahoo.language.provider.SimpleLinguisticsProvider"); + addSimpleComponent(SIMPLE_LINGUISTICS_PROVIDER); addSimpleComponent("com.yahoo.container.jdisc.SslKeyStoreFactoryProvider"); addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); @@ -280,7 +303,9 @@ public final class ContainerCluster } public final void addComponent(Component<?, ?> component) { - componentGroup.addComponent(component); + if (clusterVerifier.acceptComponent(component)) { + componentGroup.addComponent(component); + } } public final void addComponents(Collection<Component<?, ?>> components) { @@ -354,16 +379,22 @@ public final class ContainerCluster return Collections.unmodifiableList(containers); } + public void addContainer(Container container) { + if ( ! clusterVerifier.acceptContainer(container)) { + throw new IllegalArgumentException("Cluster " + name + " does not accept container " + container); + } + container.setClusterName(name); + container.setProp("clustername", name) + .setProp("index", this.containers.size()); + setRotations(container, getRotations(), getGlobalServiceId(), name); + container.setProp("activeRotation", Boolean.toString(getActiveRotation())); + containers.add(container); + } + public void addContainers(Collection<Container> containers) { - int index = this.containers.size(); for (Container container : containers) { - container.setClusterName(name); - container.setProp("clustername", name) - .setProp("index", index++); - setRotations(container, getRotations(), getGlobalServiceId(), name); - container.setProp("activeRotation", Boolean.toString(getActiveRotation())); + addContainer(container); } - this.containers.addAll(containers); } private Optional<String> getGlobalServiceId() { @@ -387,10 +418,10 @@ public final class ContainerCluster private boolean zoneHasActiveRotation(Zone zone) { return getDeploymentSpec() - .flatMap(spec -> spec.zones().stream() - .filter(dz -> dz.matches(zone.environment(), Optional.of(zone.region()))) + .flatMap(spec -> spec.steps().stream() + .filter(dz -> dz.deploysTo(zone.environment(), Optional.of(zone.region()))) .findFirst()) - .map(DeploymentSpec.DeclaredZone::active) + .map(step -> ((DeploymentSpec.DeclaredZone)step).active()) .orElse(false); } @@ -414,10 +445,6 @@ public final class ContainerCluster } } - public void addContainer(Container container) { - addContainers(Collections.singletonList(container)); - } - public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { if (this.processingChains != null) throw new IllegalStateException("ProcessingChains should only be set once."); @@ -552,7 +579,12 @@ public final class ContainerCluster allJersey1Handlers().forEach(handler -> builder.handlers.putAll(DiscBindingsConfigGenerator.generate(handler)) ); - } + } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + clusterVerifier.getConfig(builder); + } private Stream<JerseyHandler> allJersey1Handlers() { return restApiGroup.getComponents().stream().flatMap(streamOf(RestApi::getJersey1Handler)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerClusterVerifier.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerClusterVerifier.java new file mode 100644 index 00000000000..36ccf95d803 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerClusterVerifier.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.model.container; + +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.vespa.model.container.component.Component; + +/** + * + * Operations on a container cluster are verified against this verifier + * This can be used for ignoring, consitency checking. + * + * * @author baldersheim + * + */ +public interface ContainerClusterVerifier { + /** + * Return true if you accept the component. + * @param component name of component that wants to be added + * @return true if you accept it + */ + boolean acceptComponent(Component component); + + /** + * Return true if you accept the container. + * @param container container to add + * @return true if you accept it + */ + boolean acceptContainer(Container container); + + /** + * Produce threadpool config + */ + void getConfig(ThreadpoolConfig.Builder builder); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java deleted file mode 100644 index f6ed6c2eb7d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.container.jdisc.config.MetricDefaultsConfig; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.component.AccessLogComponent; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; -import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.FilterChains; -import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.JettyHttpServer; -import org.w3c.dom.Element; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.yahoo.collections.CollectionUtil.first; -import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme; -import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; - -/** - * @author tonytv - */ -public final class ManhattanContainerModelBuilder extends ContainerModelBuilder { - - static final String MANHATTAN_FILE_NAME_PATTERN = Defaults.getDefaults().vespaHome() + "logs/jdisc_core/access.%Y-%m-%d-%H"; - static final String MANHATTAN_ROTATION_INTERVAL = "0 60 ..."; - static final RotateScheme.Enum MANHATTAN_ROTATION_SCHEME = RotateScheme.DATE; - static final String MANHATTAN_SYMLINK_NAME = "access"; - - public interface BundleFiles { - // TODO: move constants to the DH code base. - Set<Path> dhBundles = new HashSet<>(Arrays.asList( - Paths.get("apache_avro/avro.jar"), - Paths.get("apache_avro/commons-compress.jar"), - Paths.get("apache_avro/paranamer.jar"), - Paths.get("apache_avro/jackson-core-asl.jar"), - Paths.get("apache_avro/jackson-mapper-asl.jar"), - Paths.get("dh_rainbow_client_api_java.jar"), - Paths.get("dh_rainbow_util_batch_java.jar"), - Paths.get("dh_rainbow_util_java.jar"))); - } - - private final int httpPort; - private JettyHttpServer jettyHttpServer; - - public ManhattanContainerModelBuilder(int httpPort) { - super(true, Networking.enable); - this.httpPort = httpPort; - } - - @Override - protected void addBundlesForPlatformComponents(ContainerCluster cluster) { - super.addBundlesForPlatformComponents(cluster); - BundleFiles.dhBundles.forEach( - bundleFile -> cluster.addPlatformBundle(absoluteBundlePath(bundleFile))); - } - - @Override - protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) { - cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); - } - - @Override - protected void addAccessLogs(ContainerCluster cluster, Element spec) { - warnIfAccessLogsDefined(spec); - - checkNotNull(jettyHttpServer, "addHttp must be called first"); - cluster.addComponent(createManhattanAccessLog()); - } - - private Component createManhattanAccessLog() { - return new AccessLogComponent(AccessLogComponent.AccessLogType.yApacheAccessLog, - MANHATTAN_FILE_NAME_PATTERN, - MANHATTAN_ROTATION_INTERVAL, - MANHATTAN_ROTATION_SCHEME, - MANHATTAN_SYMLINK_NAME); - } - - private void warnIfAccessLogsDefined(Element spec) { - List<Element> accessLogElements = getAccessLogElements(spec); - if (!accessLogElements.isEmpty()) { - logManhattanInfo("Ignoring " + accessLogElements.size() + - " access log elements in services.xml, using default yapache access logging instead."); - } - } - - @Override - protected void addDefaultHandlers(ContainerCluster cluster) { - addDefaultHandlersExceptStatus(cluster); - } - - @Override - protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { - addStatusHandlerForJDiscStatusPackage(cluster, "status.html"); //jdisc_status - addStatusHandlerForJDiscStatusPackage(cluster, "akamai"); //jdisc_akamai - } - - private static void addStatusHandlerForJDiscStatusPackage(ContainerCluster cluster, String name) { - cluster.addComponent( - new FileStatusHandlerComponent(name + "-status-handler", Defaults.getDefaults().vespaHome() + "libexec/jdisc/" + name, - "http://*/" + name, "https://*/" + name)); - } - - @Override - protected void addHttp(Element spec, ContainerCluster cluster) { - super.addHttp(spec, cluster); - ensureHasHttp(cluster); - ensureOneHttpServer(cluster.getHttp()); - } - - private void ensureHasHttp(ContainerCluster cluster) { - if (cluster.getHttp() == null) - cluster.setHttp(createHttp()); - } - - private Http createHttp() { - Http http = new Http(Collections.<Http.Binding>emptyList()); - http.setFilterChains(new FilterChains(http)); - return http; - } - - private void ensureOneHttpServer(Http http) { - if (http.getHttpServer() == null || http.getHttpServer().getConnectorFactories().isEmpty()) { - JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("main-http-server")); - http.setHttpServer(jettyHttpServer); - ConnectorFactory connectorFactory = new ConnectorFactory("main-http-connector", - httpPort, null); - http.getHttpServer().addConnector(connectorFactory); - } else { - removeAllButOneConnector(http.getHttpServer()); - ConnectorFactory connectorFactory = first(http.getHttpServer().getConnectorFactories()); - connectorFactory.setListenPort(httpPort); - } - jettyHttpServer = http.getHttpServer(); - } - - private void removeAllButOneConnector(JettyHttpServer jettyHttpServer) { - int removed = 0; - - if (jettyHttpServer.getConnectorFactories().size() > 1) { - for (int i = jettyHttpServer.getConnectorFactories().size() - 1; i > 0; i--) { - ConnectorFactory c = jettyHttpServer.getConnectorFactories().get(i); - jettyHttpServer.removeConnector(c); - ++removed; - } - } - - if (removed > 0) { - logManhattanInfo("Using only the first http server " + jettyHttpServer.getConnectorFactories().get(0).getName()); - } - } - - private static <E> List<E> tail(List<E> list) { - return list.subList(1, list.size()); - } - - private void logManhattanInfo(String message) { - log.log(Level.INFO, "[Manhattan] " + message); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 35c07ad5f60..561464103c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -4,10 +4,8 @@ package com.yahoo.vespa.model.content.cluster; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; @@ -28,6 +26,7 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerClusterVerifier; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import com.yahoo.vespa.model.container.Container; @@ -417,7 +416,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri } private ContainerCluster createClusterControllers(AbstractConfigProducer parent, Collection<HostResource> hosts, String name, boolean multitenant) { - ContainerCluster clusterControllers = new ContainerCluster(parent, name, name); + ContainerCluster clusterControllers = new ContainerCluster(parent, name, name, new ClusterControllerClusterVerifier()); List<Container> containers = new ArrayList<>(); // Add a cluster controller on each config server (there is always at least one). if (clusterControllers.getContainers().isEmpty()) { @@ -595,6 +594,8 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri if (zone.region().equals(RegionName.from("us-west-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) if (zone.region().equals(RegionName.from("us-central-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) if (zone.region().equals(RegionName.from("eu-west-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) + if (zone.region().equals(RegionName.from("ap-northeast-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) + if (zone.region().equals(RegionName.from("ap-northeast-2"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java index 687531cbe11..2a1e5d70653 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -85,7 +85,8 @@ public class Dispatch extends AbstractService implements SearchInterface, public int getFrtPort() { return getRelativePort(0); } public int getDispatchPort() { return getRelativePort(1); } - public @Override int getHealthPort() { return getRelativePort(2); } + @Override + public int getHealthPort() { return getRelativePort(2); } /** * Twice the default of the number of threads in the container. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index 9259bd31018..e4940997ac8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -4,6 +4,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.config.search.core.ProtonConfig; import static java.lang.Long.min; +import static java.lang.Integer.max; /** * Tuning of proton config for a search node based on the node flavor of that node. @@ -23,7 +24,8 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { @Override public void getConfig(ProtonConfig.Builder builder) { tuneDiskWriteSpeed(builder); - tuneDocumentStoreMaxFileSize(builder); + tuneDocumentStoreMaxFileSize(builder.summary.log); + tuneDocumentStoreNumThreads(builder.summary.log); tuneFlushStrategyMemoryLimits(builder.flush.memory); tuneFlushStrategyTlsSize(builder.flush.memory); } @@ -34,7 +36,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { } } - private void tuneDocumentStoreMaxFileSize(ProtonConfig.Builder builder) { + private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { double memoryGb = nodeFlavor.getMinMainMemoryAvailableGb(); long fileSizeBytes = 4 * GB; if (memoryGb <= 12.0) { @@ -44,7 +46,11 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { } else if (memoryGb <= 64.0) { fileSizeBytes = 1 * GB; } - builder.summary.log.maxfilesize(fileSizeBytes); + builder.maxfilesize(fileSizeBytes); + } + + private void tuneDocumentStoreNumThreads(ProtonConfig.Summary.Log.Builder builder) { + builder.numthreads(max(8, (int)nodeFlavor.getMinCpuCores()/2)); } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { diff --git a/config-model/src/main/perl/deploy b/config-model/src/main/perl/deploy index 6c165951ab0..dc7b2132792 100755 --- a/config-model/src/main/perl/deploy +++ b/config-model/src/main/perl/deploy @@ -72,7 +72,7 @@ readConfFile(); use strict; use warnings; use feature qw(switch say); -use vars qw/ $opt_h $opt_n $opt_v $opt_f $opt_t $opt_1 $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /; +use vars qw/ $opt_c $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /; use Env qw($HOME); use JSON; use Getopt::Std; @@ -99,15 +99,20 @@ my $environment = "prod"; my $region = "default"; my $instance = "default"; my $version = "v2"; +my $configserver = ""; my $port = "19071"; -getopts('fhnt:v1e:E:r:a:i:p:HR:'); +getopts('c:fhnt:ve:E:r:a:i:p:HR:'); if ($opt_h) { usage(); exit 0; } +if ($opt_c) { + $configserver = $opt_c; +} + if ($opt_e) { $tenant = $opt_e; } @@ -132,13 +137,7 @@ if ($opt_p) { $port = $opt_p; } -# Use v1 of application API -if ($opt_1) { - $version = "v1"; - $pathPrefix = "/application/v1/session"; -} else { - $pathPrefix = "/application/v2/tenant/$tenant/session"; -} +$pathPrefix = "/application/v2/tenant/$tenant/session"; create_cloudconfig_dir(); @@ -303,7 +302,13 @@ sub fetch_directory { sub get_configsource_url { my ($command) = @_; - my @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`); + my @configsources; + if ($configserver and $configserver ne "") { + @configsources = ('http://' . $configserver . ':' . $port . '/'); + } else { + @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`); + } + my $configsource_url = shift(@configsources); if (!$configsource_url) { die "Could not get url to config server, make sure that VESPA_HOME and services.addr_configserver is set\n"; diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index c8bd11d7184..d34255c7127 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -21,11 +21,19 @@ Staging = element staging { text } -Prod = - element prod { - attribute global-service-id { text }?, - element region { - attribute active { xsd:boolean }, - text - }* - } +Prod = element prod { + attribute global-service-id { text }? & + Region* & + Delay* +} + +Region = element region { + attribute active { xsd:boolean }, + text +} + +Delay = element delay { + attribute hours { xsd:long }? & + attribute minutes { xsd:long }? & + attribute seconds { xsd:long }? +} diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index cd7e0802572..2b231e0cda2 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -41,4 +41,15 @@ rankprofile[3].fef.property[2].value "tensor(x[2],y[])" rankprofile[3].fef.property[3].name "vespa.type.attribute.f3" rankprofile[3].fef.property[3].value "tensor(x{})" rankprofile[3].fef.property[4].name "vespa.type.attribute.f4" -rankprofile[3].fef.property[4].value "tensor(x[10],y[20])"
\ No newline at end of file +rankprofile[3].fef.property[4].value "tensor(x[10],y[20])" +rankprofile[4].name "profile3" +rankprofile[4].fef.property[0].name "vespa.rank.firstphase" +rankprofile[4].fef.property[0].value "rankingExpression(firstphase)" +rankprofile[4].fef.property[1].name "rankingExpression(firstphase).rankingScript" +rankprofile[4].fef.property[1].value "tensor(i[10])(i) * attribute(f4)" +rankprofile[4].fef.property[2].name "vespa.type.attribute.f2" +rankprofile[4].fef.property[2].value "tensor(x[2],y[])" +rankprofile[4].fef.property[3].name "vespa.type.attribute.f3" +rankprofile[4].fef.property[3].value "tensor(x{})" +rankprofile[4].fef.property[4].name "vespa.type.attribute.f4" +rankprofile[4].fef.property[4].value "tensor(x[10],y[20])"
\ No newline at end of file diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 8b779147ddd..ef29ed80a2e 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -33,4 +33,12 @@ search tensor { } + rank-profile profile3 { + + first-phase { + expression: tensor(i[10])(i) * attribute(f4) + } + + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java index 701a7fe0d30..cb4728a5c68 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java @@ -1,13 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.junit.Test; import java.util.Map; +import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * @author gjoranv @@ -15,6 +18,28 @@ import static org.junit.Assert.assertEquals; public class MetricSetTest { @Test + public void metrics_from_children_are_added() { + MetricSet child1 = new MetricSet("child1", ImmutableList.of(new Metric("child1_metric"))); + MetricSet child2 = new MetricSet("child2", ImmutableList.of(new Metric("child2_metric"))); + MetricSet parent = new MetricSet("parent", emptyList(), ImmutableList.of(child1, child2)); + + Map<String, Metric> parentMetrics = parent.getMetrics(); + assertEquals(2, parentMetrics.size()); + assertNotNull(parentMetrics.get("child1_metric")); + assertNotNull(parentMetrics.get("child2_metric")); + } + + @Test + public void adding_the_same_child_set_twice_has_no_effect() { + MetricSet child = new MetricSet("child", ImmutableList.of(new Metric("child_metric"))); + MetricSet parent = new MetricSet("parent", emptyList(), ImmutableList.of(child, child)); + + Map<String, Metric> parentMetrics = parent.getMetrics(); + assertEquals(1, parentMetrics.size()); + assertNotNull(parentMetrics.get("child_metric")); + } + + @Test public void internal_metrics_take_precedence_over_metrics_from_children() { String METRIC_NAME = "metric1"; String COMMON_DIMENSION_KEY = "commonKey"; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index bdd203236f9..859dd15e040 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -17,16 +17,20 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerClusterVerifier; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import org.junit.Test; +import java.util.Collection; import java.util.Iterator; import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> @@ -38,14 +42,14 @@ public class ContainerClusterTest { ContainerCluster cluster = newContainerCluster(); cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); assertEquals(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD, - getMetricDefaultsConfig(cluster).factory()); + getMetricDefaultsConfig(cluster).factory()); } @Test public void requireThatDefaultMetricConsumerFactoryMatchesConfigDefault() { ContainerCluster cluster = newContainerCluster(); assertEquals(new MetricDefaultsConfig(new MetricDefaultsConfig.Builder()).factory(), - getMetricDefaultsConfig(cluster).factory()); + getMetricDefaultsConfig(cluster).factory()); } @Test @@ -71,7 +75,8 @@ public class ContainerClusterTest { @Test public void requreThatWeCanGetTheZoneConfig() { DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(true).build()) - .zone(new Zone(SystemName.cd, Environment.test, RegionName.from("some-region"))).build(); + .zone(new Zone(SystemName.cd, Environment.test, RegionName.from("some-region"))) + .build(); MockRoot root = new MockRoot("foo", state); ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); @@ -83,13 +88,29 @@ public class ContainerClusterTest { } private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster) { - return createContainerCluster(isHosted, isCombinedCluster, Optional.empty()); + return createContainerCluster(isHosted, isCombinedCluster, Optional.empty(), Optional.empty()); } - private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, + + private ContainerCluster createClusterControllerCluster() { + return createContainerCluster(false, false, new ClusterControllerClusterVerifier()); + } + + private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, ContainerClusterVerifier extraComponents) { + return createContainerCluster(isHosted, isCombinedCluster, Optional.empty(), Optional.of(extraComponents)); + } + + private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, Optional<Integer> memoryPercentage) { + return createContainerCluster(isHosted, isCombinedCluster, memoryPercentage, Optional.empty()); + } + private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, + Optional<Integer> memoryPercentage, Optional<ContainerClusterVerifier> extraComponents) { DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(); MockRoot root = new MockRoot("foo", state); - ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); + + ContainerCluster cluster = extraComponents.isPresent() + ? new ContainerCluster(root, "container0", "container1", extraComponents.get()) + : new ContainerCluster(root, "container0", "container1"); if (isCombinedCluster) cluster.setHostClusterId("test-content-cluster"); cluster.setMemoryPercentage(memoryPercentage); @@ -151,9 +172,19 @@ public class ContainerClusterTest { } @Test + public void testContainerClusterMaxThreads() { + ContainerCluster cluster = createContainerCluster(false, false); + addContainer(cluster, "c1","host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + } + + @Test public void testClusterControllerResourceUsage() { - boolean isHosted = false; - ContainerCluster cluster = createContainerCluster(isHosted, false); + ContainerCluster cluster = createClusterControllerCluster(); addClusterController(cluster, "host-c1"); assertEquals(1, cluster.getContainers().size()); ClusterControllerContainer container = (ClusterControllerContainer) cluster.getContainers().get(0); @@ -163,12 +194,45 @@ public class ContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - container.getConfig(tpBuilder); + cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(10, threadpoolConfig.maxthreads()); } @Test + public void testThatYouCanNotAddNonClusterControllerContainerToClusterControllerCluster() { + ContainerCluster cluster = createClusterControllerCluster(); + addClusterController(cluster, "host-c1"); + try { + addContainer(cluster, "c2", "host-c2"); + assertTrue(false); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Cluster container1 does not accept container com.yahoo.vespa.model.container.Container")); + } + } + + @Test + public void testThatLinguisticsIsExcludedForClusterControllerCluster() { + ContainerCluster cluster = createClusterControllerCluster(); + addClusterController(cluster, "host-c1"); + assertFalse(contains("com.yahoo.language.provider.SimpleLinguisticsProvider", cluster.getAllComponents())); + } + + @Test + public void testThatLinguisticsIsIncludedForNonClusterControllerClusters() { + ContainerCluster cluster = createContainerCluster(false, false); + addClusterController(cluster, "host-c1"); + assertTrue(contains("com.yahoo.language.provider.SimpleLinguisticsProvider", cluster.getAllComponents())); + } + + private static boolean contains(String componentId, Collection<Component<?, ?>> componentList) { + for (Component<?, ?> component : componentList) + if (component.getClassId().toId().getName().equals(componentId)) + return true; + return false; + } + + @Test public void requireThatJvmArgsControlWorksForHostedAndNot() { verifyJvmArgs(true, false); verifyJvmArgs(true, true); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index b59f7c67abe..67e1221e61c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -45,6 +45,19 @@ public class NodeFlavorTuningTest { } @Test + public void require_that_documentstore_numthreads_is_based_on_num_cores() { + assertDocumentStoreNumThreads(8, 0); + assertDocumentStoreNumThreads(8, 1.0); + assertDocumentStoreNumThreads(8, 3.0); + assertDocumentStoreNumThreads(8, 4.0); + assertDocumentStoreNumThreads(8, 8.0); + assertDocumentStoreNumThreads(12, 24.0); + assertDocumentStoreNumThreads(16, 32.0); + assertDocumentStoreNumThreads(24, 48.0); + assertDocumentStoreNumThreads(32, 64.0); + } + + @Test public void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { assertFlushStrategyMemory(512 * MB, 4); assertFlushStrategyMemory(1 * GB, 8); @@ -70,6 +83,10 @@ public class NodeFlavorTuningTest { assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb).flush().memory().each().maxmemory()); } + private static void assertDocumentStoreNumThreads(int numThreads, double numCores) { + assertEquals(numThreads, configFromNumCoresSetting(numCores).summary().log().numthreads()); + } + private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize()); } @@ -89,6 +106,10 @@ public class NodeFlavorTuningTest { minMainMemoryAvailableGb(memoryGb)); } + private static ProtonConfig configFromNumCoresSetting(double numCores) { + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); + } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { flavorBuilder.name("my_flavor"); NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))); diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala deleted file mode 100644 index 4a437994656..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml - - -import org.junit.Test -import scala.xml.{PrettyPrinter, Elem} - -import ManhattanContainerModelBuilderTest._ -import com.yahoo.config.model.test.MockRoot -import org.apache.commons.io.IOUtils -import com.yahoo.vespa.model.container.ContainerCluster -import com.yahoo.vespa.model.container.component.{Component, AccessLogComponent} -import scala.collection.JavaConversions._ -import scala.reflect.ClassTag -import com.yahoo.config.model.producer.AbstractConfigProducer -import com.yahoo.osgi.provider.model.ComponentModel -import org.junit.Assert.{assertThat, assertNotNull} -import org.hamcrest.CoreMatchers.is -import com.yahoo.container.handler.VipStatusHandler -import com.yahoo.config.model.builder.xml.XmlHelper.getDocumentBuilder -import com.yahoo.vespa.model.container.search.searchchain.FederationSearcherTest -import com.yahoo.container.jdisc.config.HttpServerConfig -import com.yahoo.config.model.deploy.DeployState - -import scala.language.reflectiveCalls - -/** - * @author tonytv - */ -class ManhattanContainerModelBuilderTest { - - val emptyJDiscElement = <jdisc version="1.0" /> - - @Test - def multiple_access_logs_configured() { - val container = buildManhattanContainer( - <jdisc version="1.0"> - <accesslog type="yapache" fileNamePattern="myPattern" /> - <accesslog type="vespa" fileNamePattern="myPattern" /> - </jdisc>) - } - - @Test - def status_html_and_akamai_handlers_configured() { - val container = buildManhattanContainer(emptyJDiscElement) - - val vipStatusComponents = getComponentsWithModelClass[VipStatusHandler](container) - val ids = vipStatusComponents map { _.model.getComponentId.getName } - - assertThat(ids.toSet, is(Set("status.html-status-handler", "akamai-status-handler"))) - } - - @Test - def http_server_added_automatically() { - val container = buildManhattanContainer(emptyJDiscElement) - - assertThat(((container.getHttp.getHttpServer != null) && (container.getHttp.getHttpServer.getConnectorFactories.size() == 1)), is(true)) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getListenPort, is(httpPort)) - } - - @Test - def only_the_first_http_server_is_kept() { - val container = buildManhattanContainer( - <jdisc version="1.0"> - <http> - <server id="server1" port="123" /> - <server id="server2" port="456" /> - </http> - </jdisc>) - - assertThat(((container.getHttp.getHttpServer != null) && (container.getHttp.getHttpServer.getConnectorFactories.size() == 1)), is(true)) - assertThat(container.getHttp.getHttpServer.getComponentId.getName, is("jdisc-jetty")) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getName, is("server1")) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getListenPort, is(httpPort)) - } - - @Test - def filters_and_bindings_are_preserved() { - val container = buildManhattanContainer( - <jdisc version="1.0"> - <http> - <filtering> - <filter id="my-filter" /> - <request-chain id="my-chain"> - <filter id="my-filter" /> - <binding>http://*:123/my-binding</binding> - </request-chain> - </filtering> - <server id="server1" port="123" /> - </http> - </jdisc>) - - val binding = container.getHttp.getBindings.head - assertThat(binding.filterId.getName, is("my-chain")) - assertThat(binding.binding, is("http://*:123/my-binding")) - - val filterChains = container.getHttp.getFilterChains - assertNotNull("Missing filter", filterChains.componentsRegistry().getComponent("my-filter")) - assertNotNull("Missing chain", filterChains.allChains().getComponent("my-chain")) - } -} - -object ManhattanContainerModelBuilderTest { - type ACP = AbstractConfigProducer[_] - type COMPONENT = Component[_ <: ACP, _ <: ComponentModel] - - val httpPort = 9876 - - def getComponents[T <: COMPONENT](cluster: ContainerCluster)(implicit tag: ClassTag[T]): Iterable[T] = { - fixType(cluster.getComponentsMap.values()) collect { case c: T => c } - } - - def getComponentsWithModelClass[T <: AnyRef](cluster: ContainerCluster)(implicit tag: ClassTag[T]) = { - val className = tag.runtimeClass.getName - fixType(cluster.getAllComponents) filter { _.model.getClassId.getName == className } - } - - def modelClassIdMatches(name: String): PartialFunction[COMPONENT, COMPONENT] = { - case c: COMPONENT if c.model.getClassId.getName == name => c - } - - def fixType(components: java.util.Collection[_ <: Component[_, _]]): java.util.Collection[COMPONENT] = - components.asInstanceOf[java.util.Collection[COMPONENT]] - - def buildManhattanContainer(elem: Elem) = { - val root = new MockRoot() - val containerModel = new ManhattanContainerModelBuilder(httpPort).build(DeployState.createTestState(), null, root, domElement(elem)) - root.freezeModelTopology() - containerModel.getCluster() - } - - def xmlStringBuilder(elem: Elem) = { - val printer = new PrettyPrinter(240, 2) - val builder = new StringBuilder - builder.append("<?xml version='1.0' encoding='utf-8' ?>\n") - printer.format(elem, builder) - builder - } - - def domElement(elem: Elem) = { - val stream = IOUtils.toInputStream(xmlStringBuilder(elem)) - getDocumentBuilder.parse(stream).getDocumentElement - } -} diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index 92d59abbe53..286466eff57 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -4,7 +4,9 @@ <staging/> <prod global-service-id='qrs'> <region active='true'>us-west-1</region> + <delay hours='3'/> <region active='true'>us-central-1</region> + <delay hours='3' minutes='7' seconds='13'/> <region active='true'>us-east-3</region> </prod> </deployment> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index e1f8ef9b672..efc012a0946 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -40,21 +40,6 @@ public final class ApplicationId implements Comparable<ApplicationId> { return new ApplicationId(TenantName.from(tenant), ApplicationName.from(application), InstanceName.from(instance)); } - /** - * Creates an application id from a string on the form application:environment:region:instance - * - * @deprecated don't pass TenantName - */ - @Deprecated // TODO: Remove when no version older than 6.90 is in use - public static ApplicationId fromSerializedForm(TenantName tenant, String idString) { - String[] parts = idString.split(":"); - if (parts.length < 3) - throw new IllegalArgumentException("Application ids must be on the form tenant:application:instance, but was " + idString); - - return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); - } - - public static ApplicationId fromSerializedForm(String idString) { String[] parts = idString.split(":"); if (parts.length < 3) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index b78c4bb9f56..d777a96d97e 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -2,9 +2,6 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; - -import java.util.Optional; /** * A node's membership in a cluster. @@ -113,18 +110,6 @@ public class ClusterMembership { @Override public String toString() { return stringValue(); } - @Deprecated - // TODO: April 2017 - Remove this when no version older than 6.92 is in production - public static ClusterMembership from(String stringValue, Optional<String> dockerImage) { - return from(stringValue, dockerImage.map(DockerImage::new).map(DockerImage::tagAsVersion).orElse(Vtag.currentVersion)); - } - - @Deprecated - // TODO: April 2017 - Remove this when no version older than 6.97 is in production - public static ClusterMembership fromVersion(String stringValue, Optional<Version> vespaVersion) { - return new ClusterMembership(stringValue, vespaVersion.orElse(Vtag.currentVersion)); - } - public static ClusterMembership from(String stringValue, Version vespaVersion) { return new ClusterMembership(stringValue, vespaVersion); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 6a79f2f06ed..19fcce4b965 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -2,7 +2,6 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import java.util.Objects; import java.util.Optional; @@ -47,29 +46,9 @@ public final class ClusterSpec { public ClusterSpec changeGroup(Optional<Group> newGroup) { return new ClusterSpec(type, id, newGroup, vespaVersion); } - /** Create a specification <b>requesting</b> a cluster with these attributes */ - @Deprecated - // TODO: April 2017 - Remove this when no version older than 6.94 is used anywhere - public static ClusterSpec request(Type type, Id id, Optional<String> dockerImage) { - return requestVersion(type, id, dockerImage.map(DockerImage::new).map(DockerImage::tagAsVersion)); - } - - /** Create a specification <b>requesting</b> a cluster with these attributes */ - // TODO: April 2017 - Remove this when no version older than 6.97 is used anywhere - public static ClusterSpec requestVersion(Type type, Id id, Optional<Version> vespaVersion) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion.orElse(Vtag.currentVersion)); - } - public static ClusterSpec request(Type type, Id id, Version vespaVersion) { return new ClusterSpec(type, id, Optional.empty(), vespaVersion); } - - /** Create a specification <b>specifying</b> an existing cluster group having these attributes */ - // TODO: April 2017 - Remove this when no version older than 6.97 is used anywhere - public static ClusterSpec from(Type type, Id id, Group groupId, Optional<Version> vespaVersion) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion.orElse(Vtag.currentVersion)); - } - public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion) { return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java index d817dd4834c..40021eab88e 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.yahoo.component.Vtag; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -28,7 +27,6 @@ public class ProvisionInfo { private static final String hostSpecMembership = "membership"; private static final String hostSpecFlavor = "flavor"; private static final String hostSpecVespaVersion = "vespaVersion"; - private static final String dockerImage = "dockerImage"; private final Set<HostSpec> hosts = new LinkedHashSet<>(); @@ -84,11 +82,8 @@ public class ProvisionInfo { } private static ClusterMembership readMembership(Inspector object) { - // TODO: When no version older than 6.97 is present anywhere, remove the possibility of the version field missing (and hence also remove reading of dockerImage) return ClusterMembership.from(object.field(hostSpecMembership).asString(), - object.field(hostSpecVespaVersion).valid() ? - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString()) : - ( object.field(dockerImage).valid() ? new DockerImage(object.field(dockerImage).asString()).tagAsVersion() : Vtag.currentVersion)); + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); } private static Optional<Flavor> readFlavor(Inspector object, Optional<NodeFlavors> nodeFlavors) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java index d54b76c1b65..bd691afaf55 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java @@ -21,7 +21,6 @@ public class ProvisionInfoTest { private final HostSpec h1 = new HostSpec("host1", Optional.empty()); private final HostSpec h2 = new HostSpec("host2", Optional.empty()); private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1")))); - private final HostSpec h4 = new HostSpec("host4", Optional.of(ClusterMembership.from("container/test/1", Optional.of("docker-registry.ops.yahoo.com:4443/vespa/ci:6.42.1")))); @Test public void testProvisionInfoSerialization() throws IOException { @@ -29,7 +28,6 @@ public class ProvisionInfoTest { hosts.add(h1); hosts.add(h2); hosts.add(h3); - hosts.add(h4); ProvisionInfo info = ProvisionInfo.withHosts(hosts); assertProvisionInfo(info); } @@ -40,7 +38,6 @@ public class ProvisionInfoTest { Set<HostSpec> hostsB = new LinkedHashSet<>(); hostsB.add(h2); hostsB.add(h3); - hostsB.add(h4); ProvisionInfo infoA = ProvisionInfo.withHosts(hostsA); ProvisionInfo infoB = ProvisionInfo.withHosts(hostsB); @@ -54,11 +51,10 @@ public class ProvisionInfoTest { assertTrue(serializedInfo.getHosts().contains(h1)); assertTrue(serializedInfo.getHosts().contains(h2)); assertTrue(serializedInfo.getHosts().contains(h3)); - assertTrue(serializedInfo.getHosts().contains(h4)); assertTrue(!getHost(h1.hostname(), serializedInfo.getHosts()).membership().isPresent()); assertEquals("container/test/0", getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().stringValue()); assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().cluster().vespaVersion()); - assertEquals("docker-registry.ops.yahoo.com:4443/vespa/ci:6.42.1", getHost(h4.hostname(), serializedInfo.getHosts()).membership().get().cluster().dockerImage()); + assertEquals("docker-registry.ops.yahoo.com:4443/vespa/ci:6.73.1", getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().cluster().dockerImage()); } private HostSpec getHost(String hostname, Set<HostSpec> hosts) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index eff25aa9191..35dcf522682 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -110,7 +110,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ @SuppressWarnings({"UnusedDeclaration"}) public final void getConfigV3(Request req) { - log.log(LogLevel.SPAM, "getConfigV3"); + log.log(LogLevel.SPAM, () -> "getConfigV3"); JRTServerConfigRequest request = JRTServerConfigRequestV3.createFromRequest(req); if (isProtocolVersionSupported(request)) { preHandle(req); @@ -144,9 +144,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ private void getConfigImpl(JRTServerConfigRequest request) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy getConfig()"); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); - } + log.log(LogLevel.DEBUG, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); if (!request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); @@ -156,15 +154,11 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer try { RawConfig config = proxyServer.resolveConfig(request); if (config == null) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "No config received yet for " + request.getShortDescription() + ", not sending response"); - } + log.log(LogLevel.SPAM, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); } else if (ProxyServer.configOrGenerationHasChanged(config, request)) { returnOkResponse(request, config); } else { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "No new config for " + request.getShortDescription() + ", not sending response"); - } + log.log(LogLevel.SPAM, "No new config for " + request.getShortDescription() + ", not sending response"); } } catch (Exception e) { e.printStackTrace(); @@ -245,7 +239,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public final void setMode(Request req) { String suppliedMode = req.parameters().get(0).asString(); - log.log(LogLevel.DEBUG, "Supplied mode=" + suppliedMode); + log.log(LogLevel.DEBUG, () -> "Supplied mode=" + suppliedMode); String[] s = new String[2]; if (Mode.validModeName(suppliedMode.toLowerCase())) { proxyServer.setMode(suppliedMode); @@ -303,12 +297,12 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ @Override public void notifyTargetInvalid(Target target) { - log.log(LogLevel.DEBUG, "Target invalid " + target); + log.log(LogLevel.DEBUG, () -> "Target invalid " + target); for (Iterator<DelayedResponse> it = proxyServer.delayedResponses.responses().iterator(); it.hasNext(); ) { DelayedResponse delayed = it.next(); JRTServerConfigRequest request = delayed.getRequest(); if (request.getRequest().target().equals(target)) { - log.log(LogLevel.DEBUG, "Removing " + request.getShortDescription()); + log.log(LogLevel.DEBUG, () -> "Removing " + request.getShortDescription()); it.remove(); } } @@ -319,17 +313,17 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void returnOkResponse(JRTServerConfigRequest request, RawConfig config) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy returnOkResponse()"); request.addOkResponse(config.getPayload(), config.getGeneration(), config.getConfigMd5()); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + - ",config=" + config.getPayload() + ",generation=" + config.getGeneration()); - } + log.log(LogLevel.DEBUG, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + + ",generation=" + config.getGeneration()); + log.log(LogLevel.SPAM, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); + // TODO Catch exception for now, since the request might have been returned in CheckDelayedResponse // TODO Move logic so that all requests are returned in CheckDelayedResponse try { request.getRequest().returnRequest(); } catch (IllegalStateException e) { - log.log(LogLevel.DEBUG, "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); + log.log(LogLevel.DEBUG, () -> "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); } } diff --git a/config/src/apps/configproxy-cmd/proxycmd.cpp b/config/src/apps/configproxy-cmd/proxycmd.cpp index 794c005f485..df37333f514 100644 --- a/config/src/apps/configproxy-cmd/proxycmd.cpp +++ b/config/src/apps/configproxy-cmd/proxycmd.cpp @@ -1,8 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "proxycmd.h" -#include <iostream> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> + +#include <iostream> Flags::Flags(const Flags &) = default; Flags & Flags::operator=(const Flags &) = default; diff --git a/config/src/apps/configproxy-cmd/proxycmd.h b/config/src/apps/configproxy-cmd/proxycmd.h index a0ef02edaae..57b4d83fd42 100644 --- a/config/src/apps/configproxy-cmd/proxycmd.h +++ b/config/src/apps/configproxy-cmd/proxycmd.h @@ -1,7 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +class FRT_Supervisor; +class FRT_Target; +class FRT_RPCRequest; +class FRT_Values; struct Flags { vespalib::string method; @@ -30,9 +36,7 @@ private: void autoPrint(); public: ProxyCmd(const Flags& flags); - virtual ~ProxyCmd(); - int action(); }; diff --git a/config/src/main/java/.gitignore b/config/src/main/java/.gitignore deleted file mode 100644 index 4cb44b1b2b5..00000000000 --- a/config/src/main/java/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/deploy diff --git a/config/src/main/java/com/yahoo/jrt/.gitignore b/config/src/main/java/com/yahoo/jrt/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/config/src/main/java/com/yahoo/jrt/.gitignore +++ /dev/null diff --git a/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore b/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore +++ /dev/null diff --git a/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore b/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore +++ /dev/null diff --git a/config/src/tests/api/api.cpp b/config/src/tests/api/api.cpp index 54042fcef1f..430fdce11d7 100644 --- a/config/src/tests/api/api.cpp +++ b/config/src/tests/api/api.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <config-my.h> diff --git a/config/src/tests/configfetcher/configfetcher.cpp b/config/src/tests/configfetcher/configfetcher.cpp index 5d4ee8dcd2b..02d90352f69 100644 --- a/config/src/tests/configfetcher/configfetcher.cpp +++ b/config/src/tests/configfetcher/configfetcher.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/helper/configfetcher.h> #include <fstream> diff --git a/config/src/tests/configformat/configformat.cpp b/config/src/tests/configformat/configformat.cpp index 5ed10b81023..a1f3fb8f3a7 100644 --- a/config/src/tests/configformat/configformat.cpp +++ b/config/src/tests/configformat/configformat.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/print/fileconfigformatter.h> diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp index b11d2009ebf..c103f86f6e9 100644 --- a/config/src/tests/configholder/configholder.cpp +++ b/config/src/tests/configholder/configholder.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/configholder.h> @@ -62,7 +61,7 @@ TEST_MT_F("Require that wait is interrupted", 2, ConfigHolder) TEST_BARRIER(); } else { TEST_BARRIER(); - FastOS_Thread::Sleep(500); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); f.interrupt(); TEST_BARRIER(); } diff --git a/config/src/tests/configparser/configparser.cpp b/config/src/tests/configparser/configparser.cpp index 00e8929f64a..40c0a5b99bd 100644 --- a/config/src/tests/configparser/configparser.cpp +++ b/config/src/tests/configparser/configparser.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/configparser.h> diff --git a/config/src/tests/configuri/configuri_test.cpp b/config/src/tests/configuri/configuri_test.cpp index 83b99542ce4..12762356f6c 100644 --- a/config/src/tests/configuri/configuri_test.cpp +++ b/config/src/tests/configuri/configuri_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include "config-my.h" diff --git a/config/src/tests/file_subscription/file_subscription.cpp b/config/src/tests/file_subscription/file_subscription.cpp index c770d8da349..622610ac1b5 100644 --- a/config/src/tests/file_subscription/file_subscription.cpp +++ b/config/src/tests/file_subscription/file_subscription.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/configholder.h> diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp index deed6f15f1a..28b688c189e 100644 --- a/config/src/tests/functiontest/functiontest.cpp +++ b/config/src/tests/functiontest/functiontest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/config/config.h> #include "config-function-test.h" diff --git a/config/src/tests/legacysubscriber/legacysubscriber.cpp b/config/src/tests/legacysubscriber/legacysubscriber.cpp index 86ea70ead4f..25ff596e373 100644 --- a/config/src/tests/legacysubscriber/legacysubscriber.cpp +++ b/config/src/tests/legacysubscriber/legacysubscriber.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/helper/legacysubscriber.h> #include <fstream> diff --git a/config/src/tests/misc/configsystem.cpp b/config/src/tests/misc/configsystem.cpp index cef5a917732..e41258cc215 100644 --- a/config/src/tests/misc/configsystem.cpp +++ b/config/src/tests/misc/configsystem.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/configsystem.h> #include <vespa/defaults.h> +#include <vespa/fastos/file.h> using namespace config; diff --git a/config/src/tests/misc/misc.cpp b/config/src/tests/misc/misc.cpp index b2595599125..0d8ce21aff7 100644 --- a/config/src/tests/misc/misc.cpp +++ b/config/src/tests/misc/misc.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/configupdate.h> #include <vespa/config/common/misc.h> diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp index 664683b651f..29ba0fb852e 100644 --- a/config/src/tests/payload_converter/payload_converter.cpp +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("payload_converter"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/config/src/tests/raw_subscription/raw_subscription.cpp b/config/src/tests/raw_subscription/raw_subscription.cpp index 69cdaa3f0d3..6376c026a4d 100644 --- a/config/src/tests/raw_subscription/raw_subscription.cpp +++ b/config/src/tests/raw_subscription/raw_subscription.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/configholder.h> diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index fa6dfcca01f..3bae5ed85b1 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/misc.h> diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp index 5974279518a..52b9ec624ac 100644 --- a/config/src/tests/subscription/subscription.cpp +++ b/config/src/tests/subscription/subscription.cpp @@ -79,7 +79,7 @@ TEST_MT_F("requireThatNextUpdateReturnsWhenNotified", 2, SubscriptionFixture(Con ASSERT_TRUE(f1.sub.nextUpdate(2, 5000)); ASSERT_TRUE(timer.MilliSecsToNow() > 200.0); } else { - FastOS_Thread::Sleep(500); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); } } @@ -94,7 +94,7 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf ASSERT_TRUE(f1.sub.nextUpdate(1, 5000)); ASSERT_TRUE(timer.MilliSecsToNow() > 300.0); } else { - FastOS_Thread::Sleep(500); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); f1.sub.close(); } } diff --git a/config/src/tests/unittest/unittest.cpp b/config/src/tests/unittest/unittest.cpp index ca83dfee486..db91784ef53 100644 --- a/config/src/tests/unittest/unittest.cpp +++ b/config/src/tests/unittest/unittest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("unittest"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/config/src/vespa/config/common/configcontext.h b/config/src/vespa/config/common/configcontext.h index c00bceaef47..4f19be9a928 100644 --- a/config/src/vespa/config/common/configcontext.h +++ b/config/src/vespa/config/common/configcontext.h @@ -5,8 +5,6 @@ #include "timingvalues.h" #include "configmanager.h" #include <vespa/config/subscription/sourcespec.h> -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/stllike/hash_map.h> namespace config { diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index a73e8caf206..f15880fea01 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -3,9 +3,8 @@ #include "exceptions.h" #include "configholder.h" #include <vespa/vespalib/util/atomic.h> -#include <memory> #include <thread> -#include <chrono> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".config.common.configmanager"); diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp index fde55f257dc..e0a0b0138b9 100644 --- a/config/src/vespa/config/common/configparser.cpp +++ b/config/src/vespa/config/common/configparser.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/common/configparser.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <stdio.h> + +#include "configparser.h" #include "misc.h" +#include <vespa/vespalib/stllike/asciistream.h> namespace config { diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h index 3b1185a48fa..613dfc33d94 100644 --- a/config/src/vespa/config/common/configparser.h +++ b/config/src/vespa/config/common/configparser.h @@ -2,12 +2,12 @@ #pragma once #include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <map> #include <set> #include <vector> -#include <errno.h> -#include <stdint.h> -#include <vespa/vespalib/stllike/string.h> +#include <cerrno> +#include <cstdint> namespace config { diff --git a/config/src/vespa/config/common/exceptions.cpp b/config/src/vespa/config/common/exceptions.cpp index d19c27b5999..b5cbe7f0e06 100644 --- a/config/src/vespa/config/common/exceptions.cpp +++ b/config/src/vespa/config/common/exceptions.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "exceptions.h" +#include <vespa/vespalib/util/stringfmt.h> namespace config { diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.cpp b/config/src/vespa/config/file_acquirer/file_acquirer.cpp index 147da7caaf9..37d66c11453 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.cpp +++ b/config/src/vespa/config/file_acquirer/file_acquirer.cpp @@ -1,24 +1,28 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "file_acquirer.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> + #include <vespa/log/log.h> LOG_SETUP(".config.file_acquirer"); namespace config { RpcFileAcquirer::RpcFileAcquirer(const vespalib::string &spec) - : _orb(), + : _orb(std::make_unique<FRT_Supervisor>()), _spec(spec) { - _orb.Start(); + _orb->Start(); } vespalib::string RpcFileAcquirer::wait_for(const vespalib::string &file_ref, double timeout_s) { vespalib::string path; - FRT_Target *target = _orb.GetTarget(_spec.c_str()); - FRT_RPCRequest *req = _orb.AllocRPCRequest(); + FRT_Target *target = _orb->GetTarget(_spec.c_str()); + FRT_RPCRequest *req = _orb->AllocRPCRequest(); req->SetMethodName("waitFor"); req->GetParams()->AddString(file_ref.data(), file_ref.size()); target->InvokeSync(req, timeout_s); @@ -35,7 +39,7 @@ RpcFileAcquirer::wait_for(const vespalib::string &file_ref, double timeout_s) RpcFileAcquirer::~RpcFileAcquirer() { - _orb.ShutDown(true); + _orb->ShutDown(true); } } // namespace config diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.h b/config/src/vespa/config/file_acquirer/file_acquirer.h index ff090c9e7c0..3623fd5775d 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.h +++ b/config/src/vespa/config/file_acquirer/file_acquirer.h @@ -2,7 +2,8 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/fnet/frt/frt.h> + +class FRT_Supervisor; namespace config { @@ -22,7 +23,7 @@ struct FileAcquirer { class RpcFileAcquirer : public FileAcquirer { private: - FRT_Supervisor _orb; + std::unique_ptr<FRT_Supervisor> _orb; vespalib::string _spec; public: RpcFileAcquirer(const vespalib::string &spec); diff --git a/config/src/vespa/config/frt/frtconnection.cpp b/config/src/vespa/config/frt/frtconnection.cpp index 40e59000aef..301655d3fed 100644 --- a/config/src/vespa/config/frt/frtconnection.cpp +++ b/config/src/vespa/config/frt/frtconnection.cpp @@ -2,7 +2,10 @@ #include "frtconnection.h" #include <vespa/vespalib/util/atomic.h> #include <vespa/config/common/errorcode.h> -#include <time.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> + #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconnection"); @@ -114,4 +117,9 @@ void FRTConnection::calculateSuspension(ErrorType type) } } +FRT_RPCRequest * +FRTConnection::allocRPCRequest() { + return _supervisor.AllocRPCRequest(); +} + } diff --git a/config/src/vespa/config/frt/frtconnection.h b/config/src/vespa/config/frt/frtconnection.h index d0a8163413c..8eb963dc878 100644 --- a/config/src/vespa/config/frt/frtconnection.h +++ b/config/src/vespa/config/frt/frtconnection.h @@ -2,12 +2,14 @@ #pragma once #include <string> -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/config/common/timingvalues.h> #include "connection.h" +class FRT_Supervisor; +class FRT_Target; + namespace config { class FRTConnection : public Connection { @@ -35,10 +37,7 @@ public: FRTConnection(const vespalib::string & address, FRT_Supervisor & supervisor, const TimingValues & timingValues); ~FRTConnection(); - FRT_RPCRequest * allocRPCRequest() override { - return _supervisor.AllocRPCRequest(); - } - + FRT_RPCRequest * allocRPCRequest() override; void invoke(FRT_RPCRequest * req, double timeout, FRT_IRequestWait * waiter) override; const vespalib::string & getAddress() const override { return _address; } int64_t getSuspendedUntil() { return _suspendedUntil; } diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 70225b9dd04..4f433667afc 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -2,7 +2,8 @@ #include "frtconnectionpool.h" #include <vespa/vespalib/util/host_name.h> -#include <cstdlib> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/transport.h> namespace config { @@ -26,27 +27,27 @@ FRTConnectionPool::FRTConnectionKey::operator==(const FRTConnectionKey& right) c } FRTConnectionPool::FRTConnectionPool(const ServerSpec & spec, const TimingValues & timingValues) - : _supervisor(), + : _supervisor(std::make_unique<FRT_Supervisor>()), _selectIdx(0), _hostname("") { for (size_t i(0); i < spec.numHosts(); i++) { FRTConnectionKey key(i, spec.getHost(i)); - _connections[key].reset(new FRTConnection(spec.getHost(i), _supervisor, timingValues)); + _connections[key].reset(new FRTConnection(spec.getHost(i), *_supervisor, timingValues)); } setHostname(); - _supervisor.Start(); + _supervisor->Start(); } FRTConnectionPool::~FRTConnectionPool() { - _supervisor.ShutDown(true); + _supervisor->ShutDown(true); } void FRTConnectionPool::syncTransport() { - _supervisor.GetTransport()->sync(); + _supervisor->GetTransport()->sync(); } Connection * @@ -147,5 +148,9 @@ FRTConnectionPool::setHostname() _hostname = vespalib::HostName::get(); } +FNET_Scheduler * +FRTConnectionPool::getScheduler() { + return _supervisor->GetScheduler(); +} } diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h index d5af64bba93..ba09b9319b6 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.h +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -4,10 +4,10 @@ #include "frtconnection.h" #include "connectionfactory.h" #include <vespa/config/subscription/sourcespec.h> -#include <vespa/fnet/frt/frt.h> #include <vector> #include <string> #include <map> + namespace config { class FRTConnectionPool : public ConnectionFactory { @@ -32,7 +32,7 @@ private: int operator==(const FRTConnectionKey& right) const; }; - FRT_Supervisor _supervisor; + std::unique_ptr<FRT_Supervisor> _supervisor; int _selectIdx; vespalib::string _hostname; typedef std::map<FRTConnectionKey, FRTConnection::SP> ConnectionMap; @@ -56,7 +56,7 @@ public: */ void setHostname(const vespalib::string & hostname) { _hostname = hostname; } - FNET_Scheduler * getScheduler() override { return _supervisor.GetScheduler(); } + FNET_Scheduler * getScheduler() override; /** * Gets the hostname. diff --git a/config/src/vespa/config/frt/frtsource.h b/config/src/vespa/config/frt/frtsource.h index 653d182707b..fbcc55e09fe 100644 --- a/config/src/vespa/config/frt/frtsource.h +++ b/config/src/vespa/config/frt/frtsource.h @@ -1,15 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once - #include "connectionfactory.h" #include "frtconfigagent.h" #include "frtconfigrequestfactory.h" #include <vespa/config/common/configkey.h> #include <vespa/config/common/configrequest.h> #include <vespa/config/common/source.h> +#include <vespa/fnet/frt/invoker.h> -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/sync.h> namespace config { diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index ccc56fd51b9..b089af71782 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -3,6 +3,7 @@ #include <lz4.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".config.frt.protocol"); diff --git a/config/src/vespa/config/helper/configpoller.cpp b/config/src/vespa/config/helper/configpoller.cpp index ee9ec1bbc31..7166c561a5e 100644 --- a/config/src/vespa/config/helper/configpoller.cpp +++ b/config/src/vespa/config/helper/configpoller.cpp @@ -22,11 +22,7 @@ void ConfigPoller::run() { while (!_subscriber.isClosed()) { - try { - poll(); - } catch (const std::exception & e) { - LOG(fatal, "Fatal error while configuring: %s", e.what()); - } + poll(); } } diff --git a/config/src/vespa/config/helper/configpoller.h b/config/src/vespa/config/helper/configpoller.h index af660f6c536..889c6631738 100644 --- a/config/src/vespa/config/helper/configpoller.h +++ b/config/src/vespa/config/helper/configpoller.h @@ -32,9 +32,6 @@ private: IGenerationCallback *_genCallback; }; - } // namespace config #include "configpoller.hpp" - - diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index c62452a6415..75dcd3d3f65 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -2,9 +2,8 @@ #include "configsetsource.h" #include <vespa/config/print/asciiconfigwriter.h> -#include <vespa/config/common/misc.h> #include <vespa/config/common/exceptions.h> -#include <vespa/vespalib/stllike/asciistream.h> + #include <vespa/log/log.h> LOG_SETUP(".config.set.configsetsource"); @@ -48,7 +47,7 @@ ConfigSetSource::getConfig() void ConfigSetSource::reload(int64_t generation) { - LOG(debug, "Running update with generation(%" PRId64 ")", generation); + LOG(debug, "Running update with generation(%zd)", generation); _generation = generation; } diff --git a/configd/src/apps/sentinel/config-handler.cpp b/configd/src/apps/sentinel/config-handler.cpp index a801e405fde..1c9b53bde2b 100644 --- a/configd/src/apps/sentinel/config-handler.cpp +++ b/configd/src/apps/sentinel/config-handler.cpp @@ -152,9 +152,9 @@ ConfigHandler::terminate() } void -ConfigHandler::subscribe(const std::string & configId) +ConfigHandler::subscribe(const std::string & configId, uint64_t timeoutMS) { - _sentinelHandle = _subscriber.subscribe<SentinelConfig>(configId); + _sentinelHandle = _subscriber.subscribe<SentinelConfig>(configId, timeoutMS); } void diff --git a/configd/src/apps/sentinel/config-handler.h b/configd/src/apps/sentinel/config-handler.h index 2656a8df571..897f1e1ce3c 100644 --- a/configd/src/apps/sentinel/config-handler.h +++ b/configd/src/apps/sentinel/config-handler.h @@ -71,7 +71,7 @@ private: public: ConfigHandler(); virtual ~ConfigHandler(); - void subscribe(const std::string & configId); + void subscribe(const std::string & configId, uint64_t timeoutMS); bool terminate(); int doWork(); void updateActiveFdset(fd_set *fds, int *maxNum); diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 1f450185072..fed65c7fbab 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <sys/types.h> #include <signal.h> #include <cstring> @@ -8,7 +9,6 @@ #include <vespa/defaults.h> #include <vespa/log/log.h> LOG_SETUP("config-sentinel"); -LOG_RCSID("$Id$"); #include <vespa/config-sentinel.h> @@ -16,6 +16,8 @@ LOG_RCSID("$Id$"); using namespace config; +constexpr uint64_t CONFIG_TIMEOUT_MS = 3 * 60 * 1000; + static int sigPermanent(int sig, void(*handler)(int)); static void gracefulShutdown(int sig); @@ -65,7 +67,11 @@ main(int argc, char **argv) LOG(debug, "Reading configuration"); try { - handler.subscribe(configId); + handler.subscribe(configId, CONFIG_TIMEOUT_MS); + } catch (ConfigTimeoutException & ex) { + LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); + EV_STOPPING("config-sentinel", ex.what()); + exit(EXIT_FAILURE); } catch (InvalidConfigException& ex) { LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str()); EV_STOPPING("config-sentinel", ex.what()); @@ -142,4 +148,3 @@ sigPermanent(int sig, void(*handler)(int)) sa.sa_handler = handler; return sigaction(sig, &sa, nullptr); } - diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index 997416e5bfc..49f5275c7fe 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -8,10 +8,12 @@ import java.io.IOException; import java.io.FileReader; import java.io.StringWriter; import java.io.Writer; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; +import java.util.HashMap; +import java.util.Collections; +import java.util.Arrays; import java.util.StringTokenizer; +import java.util.stream.Collectors; /** @@ -189,15 +191,19 @@ public class CppClassBuilder implements ClassBuilder { } void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException { - for (int i = 0; i < namespaceList.length; i++) { - w.write("namespace " + namespaceList[i] + " {\n\n"); - } + w.write("namespace "); + w.write(getNestedNameSpace(namespaceList)); + w.write(" {\n"); + } + + String getNestedNameSpace(String [] namespaceList) { + return Arrays.stream(namespaceList).map(String::toString).collect(Collectors.joining("::")); } void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException { - for (int i = 0; i < namespaceList.length; i++) { - w.write("} // namespace " + namespaceList[i] + "\n\n"); - } + w.write("} // namespace "); + w.write(getNestedNameSpace(namespaceList)); + w.write("\n"); } void writeHeaderHeader(Writer w, CNode root) throws IOException { @@ -229,17 +235,21 @@ public class CppClassBuilder implements ClassBuilder { + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n" + "#define CLOUD_CONFIG_" + defineName + "_H\n" + "\n" - + "#include <vespa/config/common/configvalue.h>\n" - + "#include <vespa/config/configgen/configpayload.h>\n" + "#include <vespa/config/configgen/configinstance.h>\n" - + "#include <vespa/config/print/configdatabuffer.h>\n" + "#include <vespa/vespalib/stllike/string.h>\n" + "#include <vector>\n" + "#include <map>\n" + "\n"); + w.write("namespace config {\n"); + w.write(" class ConfigValue;\n"); + w.write(" class ConfigPayload;\n"); + w.write("}\n\n"); + w.write("namespace vespalib::slime {\n"); + w.write(" class Inspector;\n"); + w.write(" class Cursor;\n"); + w.write("}\n\n"); writeNameSpaceBegin(w, namespaceList); - w.write("\n"); - w.write("namespace internal {\n\n"); + w.write("\nnamespace internal {\n\n"); w.write("" + "/**\n" + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n" @@ -551,17 +561,18 @@ public class CppClassBuilder implements ClassBuilder { w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">"); } w.write("\n"); + w.write("#include <vespa/config/common/configvalue.h>\n"); + w.write("#include <vespa/config/configgen/configpayload.h>\n"); + w.write("#include <vespa/config/print/configdatabuffer.h>\n"); w.write("#include <vespa/config/common/configparser.h>\n"); w.write("#include <vespa/config/configgen/vector_inserter.h>\n"); w.write("#include <vespa/config/configgen/map_inserter.h>\n"); w.write("#include <vespa/vespalib/data/slime/convenience.h>\n"); w.write("#include <vespa/vespalib/data/slime/slime.h>\n"); w.write("#include <vespa/vespalib/stllike/asciistream.h>\n"); - w.write("#include <set>\n"); - w.write("\n\n"); - writeNameSpaceBegin(w, generateCppNameSpace(root)); w.write("\n"); - w.write("namespace internal {\n\n"); + writeNameSpaceBegin(w, generateCppNameSpace(root)); + w.write("\nnamespace internal {\n\n"); w.write("using ::config::ConfigParser;\n"); w.write("using ::config::InvalidConfigException;\n"); w.write("using ::config::ConfigInstance;\n"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index ab974090ee3..229f24152c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -2,10 +2,15 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; @@ -39,8 +44,12 @@ import java.io.File; import java.net.URI; import java.time.Clock; import java.time.Duration; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Logger; /** @@ -62,6 +71,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final HttpProxy httpProxy; private final Clock clock; private final DeployLogger logger = new SilentDeployLogger(); + private final ConfigserverConfig configserverConfig; + private final Environment environment; @Inject public ApplicationRepository(Tenants tenants, @@ -69,7 +80,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Curator curator, LogServerLogGrabber logServerLogGrabber, ApplicationConvergenceChecker applicationConvergenceChecker, - HttpProxy httpProxy) { + HttpProxy httpProxy, + ConfigserverConfig configserverConfig) { this.tenants = tenants; this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.curator = curator; @@ -77,6 +89,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.convergeChecker = applicationConvergenceChecker; this.httpProxy = httpProxy; this.clock = Clock.systemUTC(); + this.configserverConfig = configserverConfig; + this.environment = Environment.from(configserverConfig.environment()); } /** @@ -96,6 +110,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, timeoutBudget); tenant.getLocalSessionRepo().addSession(newSession); + + // Keep manually deployed applications on the latest version, don't change version otherwise + Version version = environment.isManuallyDeployed() ? Vtag.currentVersion : newSession.getVespaVersion(); + return Optional.of(Deployment.unprepared(newSession, tenant.getLocalSessionRepo(), tenant.getPath(), @@ -103,7 +121,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye new ActivateLock(curator, tenant.getPath()), timeout, clock, - /* already deployed, validate: */ false)); + false, // don't validate as this is already deployed + version)); } public Deployment deployFromPreparedSession(LocalSession session, ActivateLock lock, LocalSessionRepo localSessionRepo, Duration timeout) { @@ -111,7 +130,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye localSessionRepo, hostProvisioner, lock, - timeout, clock); + timeout, + clock); } /** @@ -284,17 +304,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return currentActiveApplicationSet; } - public ConfigChangeActions prepare(Tenant tenant, - long sessionId, - DeployLogger logger, - PrepareParams params) { + public ConfigChangeActions prepare(Tenant tenant, long sessionId, DeployLogger logger, PrepareParams params) { LocalSession session = getLocalSession(tenant, sessionId); ApplicationId appId = params.getApplicationId(); Optional<ApplicationSet> currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, appId); - return session.prepare(logger, - params, - currentActiveApplicationSet, - tenant.getPath()); + return session.prepare(logger, params, currentActiveApplicationSet, tenant.getPath()); } private List<ApplicationId> listApplicationIds(Tenant tenant) { @@ -320,6 +334,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session.getSessionId(); } + void redeployAllApplications(Deployer deployer) { + ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + tenants.getAllTenants().forEach(tenant -> listApplicationIds(tenant) + .forEach(applicationId -> redeployApplication(applicationId, deployer, deploymentExecutor))); + } + + private void redeployApplication(ApplicationId applicationId, Deployer deployer, ExecutorService deploymentExecutor) { + log.log(LogLevel.DEBUG, () -> "Redeploying " + applicationId); + deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30)) + .ifPresent(deployment -> deploymentExecutor.execute(() -> { + try { + deployment.activate(); + } catch (RuntimeException e) { + log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e); + } + })); + } + public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { Tenant tenant = tenants.getTenant(tenantName); return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index ddf318eed9f..e0def2b0e4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -6,7 +6,6 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Deployer; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.rpc.RpcServer; -import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.version.VersionState; /** @@ -18,21 +17,23 @@ import com.yahoo.vespa.config.server.version.VersionState; public class ConfigServerBootstrap extends AbstractComponent implements Runnable { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerBootstrap.class.getName()); + + private final ApplicationRepository applicationRepository; private final RpcServer server; private final Thread serverThread; + private final Deployer deployer; + private final VersionState versionState; // The tenants object is injected so that all initial requests handlers are - // added to the rpcserver before it starts answering rpc requests. + // added to the rpc server before it starts answering rpc requests. @SuppressWarnings("UnusedParameters") @Inject - public ConfigServerBootstrap(Tenants tenants, RpcServer server, Deployer deployer, VersionState versionState) { + public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, + Deployer deployer, VersionState versionState) { + this.applicationRepository = applicationRepository; this.server = server; - if (versionState.isUpgraded()) { - log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications"); - tenants.redeployApplications(deployer); - log.log(LogLevel.INFO, "All applications redeployed"); - } - versionState.saveNewVersion(); + this.deployer = deployer; + this.versionState = versionState; this.serverThread = new Thread(this, "configserver main"); serverThread.start(); } @@ -50,6 +51,13 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @Override public void run() { + if (versionState.isUpgraded()) { + log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to " + + versionState.currentVersion() + ". Redeploying all applications"); + applicationRepository.redeployAllApplications(deployer); + log.log(LogLevel.INFO, "All applications redeployed"); + } + versionState.saveNewVersion(); log.log(LogLevel.DEBUG, "Starting RPC server"); server.run(); log.log(LogLevel.DEBUG, "RPC server stopped"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index e378777192f..526aaf4c9c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; public class HttpProxy { + private static Logger logger = Logger.getLogger(HttpProxy.class.getName()); private final HttpFetcher fetcher; @@ -64,4 +65,5 @@ public class HttpProxy { HttpFetcher.Params params = new HttpFetcher.Params(2000); // 2_000 ms read timeout return fetcher.get(params, url); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index c27efb2caa2..0ce6fb546a2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; @@ -45,6 +46,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private final Duration timeout; private final Clock clock; private final DeployLogger logger = new SilentDeployLogger(); + + /** The Vespa version this application should run on */ + private final Version version; private boolean prepared = false; @@ -56,7 +60,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private Deployment(LocalSession session, LocalSessionRepo localSessionRepo, Path tenantPath, Optional<Provisioner> hostProvisioner, ActivateLock activateLock, - Duration timeout, Clock clock, boolean prepared, boolean validate) { + Duration timeout, Clock clock, boolean prepared, boolean validate, Version version) { this.session = session; this.localSessionRepo = localSessionRepo; this.tenantPath = tenantPath; @@ -66,20 +70,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.clock = clock; this.prepared = prepared; this.validate = validate; + this.version = version; } public static Deployment unprepared(LocalSession session, LocalSessionRepo localSessionRepo, Path tenantPath, Optional<Provisioner> hostProvisioner, ActivateLock activateLock, - Duration timeout, Clock clock, boolean validate) { + Duration timeout, Clock clock, boolean validate, Version version) { return new Deployment(session, localSessionRepo, tenantPath, hostProvisioner, activateLock, - timeout, clock, false, validate); + timeout, clock, false, validate, version); } public static Deployment prepared(LocalSession session, LocalSessionRepo localSessionRepo, Optional<Provisioner> hostProvisioner, ActivateLock activateLock, Duration timeout, Clock clock) { return new Deployment(session, localSessionRepo, null, hostProvisioner, activateLock, - timeout, clock, true, true); + timeout, clock, true, true, session.getVespaVersion()); } public Deployment setIgnoreLockFailure(boolean ignoreLockFailure) { @@ -97,12 +102,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public void prepare() { if (prepared) return; TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + session.prepare(logger, - /** Assumes that session has already set application id, see {@link com.yahoo.vespa.config.server.session.SessionFactoryImpl}. */ new PrepareParams.Builder().applicationId(session.getApplicationId()) .timeoutBudget(timeoutBudget) .ignoreValidationErrors( ! validate) - .vespaVersion(session.getVespaVersion().toString()) + .vespaVersion(version.toString()) .build(), Optional.empty(), tenantPath); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java index 84df1834712..28591f8f198 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java @@ -27,7 +27,7 @@ public class ListTenantsHandler extends HttpHandler { @Override protected HttpResponse handleGET(HttpRequest request) { - return new ListTenantsResponse(tenants.getAllTenants()); + return new ListTenantsResponse(tenants.getAllTenantNames()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 70c855f064c..b37a54c6db0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -78,7 +78,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ServerCache cache = zkClient.loadServerCache(); ModelContext modelContext = new ModelContextImpl( applicationPackage, - Optional.<Model>empty(), + Optional.empty(), permanentApplicationPackage.get().applicationPackage(), logger, configDefinitionRepo, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 3890001518a..0b742b09cdd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -48,6 +48,7 @@ public final class PrepareParams { } public static class Builder { + private boolean ignoreValidationErrors = false; private boolean dryRun = false; private ApplicationId applicationId = ApplicationId.defaultId(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 96c0f912fd4..fd18a2de997 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -87,18 +87,18 @@ public class SessionPreparer { */ public ConfigChangeActions prepare(SessionContext context, DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) { - Preparation prep = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); - prep.preprocess(); + Preparation preparation = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); + preparation.preprocess(); try { - prep.buildModels(); - prep.makeResult(); - if (!params.isDryRun()) { - prep.writeStateZK(); - prep.writeRotZK(); - prep.distribute(); - prep.reloadDeployFileDistributor(); + preparation.buildModels(); + preparation.makeResult(); + if ( ! params.isDryRun()) { + preparation.writeStateZK(); + preparation.writeRotZK(); + preparation.distribute(); + preparation.reloadDeployFileDistributor(); } - return prep.result(); + return preparation.result(); } catch (OutOfCapacityException e) { throw e; } catch (IllegalArgumentException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index e03de29a9e4..b430b78038b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -163,21 +163,5 @@ public class Tenant implements TenantHandlerProvider { curator.delete(path); } - public void redeployApplications(Deployer deployer) { - // TODO: Configurable timeout - applicationRepo.listApplications().forEach(applicationId -> redeployApplication(applicationId, deployer)); - } - - private void redeployApplication(ApplicationId applicationId, Deployer deployer) { - try { - log.log(LogLevel.DEBUG, "Redeploying " + applicationId); - deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30)) - .ifPresent(deployment -> { - deployment.prepare(); - deployment.activate(); - }); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e); - } - } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index e28c353e532..cc21616050a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -87,7 +86,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen // Note: unit tests may want to use the constructor below to avoid setting watch by calling readTenants(). this.globalComponentRegistry = globalComponentRegistry; this.curator = globalComponentRegistry.getCurator(); - metricUpdater = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); + metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(globalComponentRegistry.getTenantListener()); curator.framework().getConnectionStateListenable().addListener(this); @@ -95,19 +94,13 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen createSystemTenants(); curator.create(vespaPath); - this.directoryCache = globalComponentRegistry.getCurator().createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); + this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); directoryCache.start(); directoryCache.addListener(this); tenantsChanged(readTenants()); notifyTenantsLoaded(); } - private void notifyTenantsLoaded() { - for (TenantListener tenantListener : tenantListeners) { - tenantListener.onTenantsLoaded(); - } - } - /** * New instance containing the given tenants. This will not watch in ZooKeeper. * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} instance @@ -124,6 +117,12 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen this.tenants.putAll(addTenants(tenants)); } + private void notifyTenantsLoaded() { + for (TenantListener tenantListener : tenantListeners) { + tenantListener.onTenantsLoaded(); + } + } + // Pre-condition: tenants path needs to exist in zk private LinkedHashMap<TenantName, Tenant> addTenants(Collection<Tenant> newTenants) { LinkedHashMap<TenantName, Tenant> sessionTenants = new LinkedHashMap<>(); @@ -326,17 +325,6 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen pathChildrenExecutor.shutdown(); } - public void redeployApplications(Deployer deployer) { - Set<Tenant> allTenants = ImmutableSet.copyOf(tenants.values()); - int totalNumberOfApplications = allTenants.stream() - .mapToInt(tenant -> tenant.getApplicationRepo().listApplications().size()).sum(); - int applicationsRedeployed = 0; - for (Tenant tenant : allTenants) { - tenant.redeployApplications(deployer); - applicationsRedeployed += redeployProgress(tenant, applicationsRedeployed, totalNumberOfApplications); - } - } - public boolean checkThatTenantExists(TenantName tenant) { return tenants.containsKey(tenant); } @@ -345,16 +333,12 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen return tenants.get(tenantName); } - public Set<TenantName> getAllTenants() { + public Set<TenantName> getAllTenantNames() { return ImmutableSet.copyOf(tenants.keySet()); } - private static int redeployProgress(Tenant tenant, int applicationsRedeployed, int totalNumberOfApplications) { - int size = tenant.getApplicationRepo().listApplications().size(); - if (size > 0) { - log.log(LogLevel.INFO, String.format("Redeployed %s of %s applications", applicationsRedeployed + size, totalNumberOfApplications)); - } - return size; + public Collection<Tenant> getAllTenants() { + return ImmutableSet.copyOf(tenants.values()); } /** diff --git a/configserver/src/main/resources/logd/logd.cfg b/configserver/src/main/resources/logd/logd.cfg index a4677deb3f6..137f3742ef2 100644 --- a/configserver/src/main/resources/logd/logd.cfg +++ b/configserver/src/main/resources/logd/logd.cfg @@ -7,3 +7,4 @@ loglevel.info.forward false loglevel.event.forward false loglevel.debug.forward false loglevel.spam.forward false +remove.totalmegabytes 5000 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 29d3273a527..2443748a99c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,14 +2,27 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.HttpProxy; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; +import com.yahoo.vespa.config.server.deploy.MockDeployer; import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; +import com.yahoo.vespa.config.server.http.v2.SessionActiveHandlerTest; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; +import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRequestHandler; import com.yahoo.vespa.config.server.tenant.TestWithTenant; import com.yahoo.vespa.config.server.version.VersionState; +import org.hamcrest.core.Is; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -29,9 +42,28 @@ import static org.junit.Assert.assertTrue; * @since 5.1 */ public class ConfigServerBootstrapTest extends TestWithTenant { + private final TenantName tenant1 = TenantName.from("tenant1"); + private final TenantName tenant2 = TenantName.from("tenant2"); + + private ApplicationRepository applicationRepository; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before + public void setup() { + tenants.writeTenantPath(tenant1); + tenants.writeTenantPath(tenant2); + + applicationRepository = new ApplicationRepository(tenants, + HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()), + curator, + new LogServerLogGrabber(), + new ApplicationConvergenceChecker(), + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder())); + } + @Test public void testConfigServerBootstrap() throws Exception { File versionFile = temporaryFolder.newFile(); @@ -43,11 +75,11 @@ public class ConfigServerBootstrapTest extends TestWithTenant { assertFalse(myServer.stopped); VersionState versionState = new VersionState(versionFile); assertTrue(versionState.isUpgraded()); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tenants, rpc, (application, timeout) -> Optional.empty(), versionState); + ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(applicationRepository, rpc, (application, timeout) -> Optional.empty(), versionState); + waitUntilStarted(rpc, 60000); assertFalse(versionState.isUpgraded()); assertThat(versionState.currentVersion(), is(versionState.storedVersion())); assertThat(IOUtils.readAll(new FileReader(versionFile)), is(versionState.currentVersion().toSerializedForm())); - waitUntilStarted(rpc, 60000); assertTrue(rpc.started); assertFalse(rpc.stopped); bootstrap.deconstruct(); @@ -55,6 +87,16 @@ public class ConfigServerBootstrapTest extends TestWithTenant { assertTrue(rpc.stopped); } + @Test + public void testTenantRedeployment() throws Exception { + MockDeployer deployer = new MockDeployer(); + Tenant tenant = tenants.getTenant(tenant1); + ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName()); + tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit(); + applicationRepository.redeployAllApplications(deployer); + assertThat(deployer.lastDeployed, Is.is(id)); + } + private void waitUntilStarted(MockRpc server, long timeout) throws InterruptedException { long start = System.currentTimeMillis(); while ((System.currentTimeMillis() - start) < timeout) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index a1b8b4192c8..7c2b36d4d72 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -152,7 +152,8 @@ public class DeployTester { curator, new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher())); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder())); return applicationRepository.deployFromLocalActive(id, Duration.ofSeconds(60)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 6eb730d11dc..199d592deb5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -65,7 +66,8 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { new MockCurator(), new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); pathPrefix = createPath(idTenant1, Zone.defaultZone()); baseUrl = baseServer + pathPrefix; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c34cb565878..330c13892ad 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.FilesApplicationPackage; @@ -114,7 +115,8 @@ public class ApplicationHandlerTest { new MockCurator(), logServerLogGrabber, convergeChecker, - httpProxy)); + httpProxy, + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } private ApplicationHandler createApplicationHandler(Tenants tenants) { @@ -127,7 +129,8 @@ public class ApplicationHandlerTest { new MockCurator(), new LogServerLogGrabber(), new ApplicationConvergenceChecker(stateApiFactory), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 2d76e9aa2cc..f5bac7b0abc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; @@ -183,7 +184,8 @@ public class SessionActiveHandlerTest extends SessionActiveHandlerTestBase { curator, new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } public static class MockProvisioner implements Provisioner { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 7f5ab96d76a..caef49e922f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; @@ -178,6 +179,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { new MockCurator(), new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 107297743e1..04d8f35b2d4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -252,7 +252,8 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { new MockCurator(), new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } public HttpRequest post() throws FileNotFoundException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index f10a7ffd502..6ca7a2922b6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -373,7 +373,8 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { new MockCurator(), new LogServerLogGrabber(), new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()))); + new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()))); } private TestTenantBuilder addTenant(TenantName tenantName, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java index 31e8d545b7a..a2b4335d4ac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java @@ -37,7 +37,7 @@ import static org.junit.Assert.fail; public class TenantsTestCase extends TestWithCurator { private Tenants tenants; - TestComponentRegistry globalComponentRegistry; + private TestComponentRegistry globalComponentRegistry; private TenantRequestHandlerTest.MockReloadListener listener; private MockTenantListener tenantListener; private final TenantName tenant1 = TenantName.from("tenant1"); @@ -80,19 +80,19 @@ public class TenantsTestCase extends TestWithCurator { @Test public void testTenantListenersNotified() throws Exception { tenants.writeTenantPath(tenant3); - assertThat("tenant3 not the last created tenant. Tenants: " + tenants.getAllTenants() + ", /config/v2/tenants: " + readZKChildren("/config/v2/tenants"), tenantListener.tenantCreatedName, is(tenant3)); + assertThat("tenant3 not the last created tenant. Tenants: " + tenants.getAllTenantNames() + ", /config/v2/tenants: " + readZKChildren("/config/v2/tenants"), tenantListener.tenantCreatedName, is(tenant3)); tenants.deleteTenant(tenant2); - assertFalse(tenants.getAllTenants().contains(tenant2)); + assertFalse(tenants.getAllTenantNames().contains(tenant2)); assertThat(tenantListener.tenantDeletedName, is(tenant2)); } @Test public void testAddTenant() throws Exception { - Set<TenantName> allTenants = tenants.getAllTenants(); + Set<TenantName> allTenants = tenants.getAllTenantNames(); assertTrue(allTenants.contains(tenant1)); assertTrue(allTenants.contains(tenant2)); tenants.writeTenantPath(tenant3); - allTenants = tenants.getAllTenants(); + allTenants = tenants.getAllTenantNames(); assertTrue(allTenants.contains(tenant1)); assertTrue(allTenants.contains(tenant2)); assertTrue(allTenants.contains(tenant3)); @@ -108,7 +108,7 @@ public class TenantsTestCase extends TestWithCurator { public void testRemove() throws Exception { assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(tenant1))); tenants.deleteTenant(tenant1); - assertFalse(tenants.getAllTenants().contains(tenant1)); + assertFalse(tenants.getAllTenantNames().contains(tenant1)); } @Test @@ -120,14 +120,14 @@ public class TenantsTestCase extends TestWithCurator { newTenants.add(tenant2); newTenants.add(defaultTenant); tenants.tenantsChanged(newTenants); - Set<TenantName> allTenants = tenants.getAllTenants(); + Set<TenantName> allTenants = tenants.getAllTenantNames(); assertTrue(allTenants.contains(tenant2)); assertEquals("default", defaultTenant.value()); assertTrue(allTenants.contains(defaultTenant)); assertFalse(allTenants.contains(tenant1)); newTenants.clear(); tenants.tenantsChanged(newTenants); - allTenants = tenants.getAllTenants(); + allTenants = tenants.getAllTenantNames(); assertFalse(allTenants.contains(tenant1)); assertFalse(allTenants.contains(tenant2)); assertFalse(allTenants.contains(defaultTenant)); @@ -138,7 +138,7 @@ public class TenantsTestCase extends TestWithCurator { newTenants.add(foo); newTenants.add(bar); tenants.tenantsChanged(newTenants); - allTenants = tenants.getAllTenants(); + allTenants = tenants.getAllTenantNames(); assertTrue(allTenants.contains(tenant2)); assertTrue(allTenants.contains(foo)); assertTrue(allTenants.contains(bar)); @@ -149,13 +149,13 @@ public class TenantsTestCase extends TestWithCurator { TestComponentRegistry reg = new TestComponentRegistry.Builder().curator(curator).build(); Tenants t = new Tenants(reg, Metrics.createTestMetrics()); try { - assertTrue(t.getAllTenants().contains(TenantName.defaultName())); + assertTrue(t.getAllTenantNames().contains(TenantName.defaultName())); reg.getCurator().framework().create().forPath(tenants.tenantZkPath(TenantName.from("newTenant"))); // Poll for the watcher to pick up the tenant from zk, and add it int tries=0; while(true) { if (tries > 500) fail("Didn't react on watch"); - if (t.getAllTenants().contains(TenantName.from("newTenant"))) { + if (t.getAllTenantNames().contains(TenantName.from("newTenant"))) { return; } tries++; @@ -166,14 +166,5 @@ public class TenantsTestCase extends TestWithCurator { } } - @Test - public void testTenantRedeployment() throws Exception { - MockDeployer deployer = new MockDeployer(); - Tenant tenant = tenants.getTenant(tenant1); - ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName()); - tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit(); - tenants.redeployApplications(deployer); - assertThat(deployer.lastDeployed, is(id)); - } } diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index 92b68e8a1ae..ee6da8f92b2 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -3,7 +3,6 @@ package com.yahoo.container; import com.yahoo.component.AbstractComponent; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.config.FileReference; import com.yahoo.container.core.config.BundleLoader; import com.yahoo.container.osgi.AbstractRpcAdaptor; import com.yahoo.container.osgi.ContainerRpcAdaptor; @@ -14,9 +13,7 @@ import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.Osgi; import com.yahoo.vespa.config.ConfigTransformer; -import com.yahoo.vespa.config.ConfigTransformer.PathAcquirer; -import java.nio.file.Path; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -145,15 +142,6 @@ public class Container { setPathAcquirer(fileAcquirer); } - /** Just a helper method to return a useful host to bind to. */ - public static String bindHostName(String host) { - if ("".equals(host)) { - return "0.0.0.0"; - } else { - return host; - } - } - /** * Only for internal use. */ @@ -167,16 +155,13 @@ public class Container { setPathAcquirer(fileAcquirer); } - private void setPathAcquirer(final FileAcquirer fileAcquirer) { - ConfigTransformer.setPathAcquirer(new PathAcquirer() { - @Override - public Path getPath(FileReference fileReference) { - try { - return fileAcquirer.waitFor(fileReference, 15, TimeUnit.MINUTES).toPath(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } + private static void setPathAcquirer(final FileAcquirer fileAcquirer) { + ConfigTransformer.setPathAcquirer(fileReference -> { + try { + return fileAcquirer.waitFor(fileReference, 15, TimeUnit.MINUTES).toPath(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); } }); } diff --git a/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore b/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore +++ /dev/null diff --git a/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore b/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore +++ /dev/null diff --git a/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore b/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore +++ /dev/null diff --git a/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore b/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore +++ /dev/null diff --git a/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore b/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore +++ /dev/null diff --git a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala index a1b8167171a..efa094cac11 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala @@ -3,11 +3,14 @@ package com.yahoo.container.di import config.Subscriber -import java.util.logging.Logger +import java.util.logging.{Level, Logger} + import com.yahoo.log.LogLevel import ConfigRetriever._ + import annotation.tailrec import com.yahoo.config.ConfigInstance + import scala.collection.JavaConversions._ import com.yahoo.vespa.config.ConfigKey @@ -77,7 +80,14 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], componentSubscriber.close() componentSubscriberKeys = keys - componentSubscriber = subscribe(keys) + try { + componentSubscriber = subscribe(keys) + } catch { + case e: Throwable => + log.log(Level.WARNING, s"Failed setting up subscriptions for component configs: ${e.getMessage}") + log.log(Level.WARNING, s"Config keys: $keys") + throw e + } } } diff --git a/container-di/src/main/scala/com/yahoo/container/di/Container.scala b/container-di/src/main/scala/com/yahoo/container/di/Container.scala index 28d99f89d73..cdec5dca04d 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/Container.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/Container.scala @@ -1,20 +1,26 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di -import com.yahoo.container.di.ConfigRetriever.{ComponentsConfigs, BootstrapConfigs} -import com.yahoo.container.di.componentgraph.core.{JerseyNode, ComponentGraph, ComponentNode} -import com.yahoo.container.di.config.{RestApiContext, SubscriberFactory} -import Container._ +import java.util.{IdentityHashMap, Random} +import java.util.logging.{Level, Logger} -import scala.collection.JavaConversions._ -import scala.math.max +import com.google.inject.{Guice, Injector} import com.yahoo.config._ -import com.yahoo.vespa.config.ConfigKey -import java.util.IdentityHashMap -import java.util.logging.Logger +import com.yahoo.config.subscription.ConfigInterruptedException import com.yahoo.container.bundle.BundleInstantiationSpecification -import com.google.inject.{Injector, Guice} +import com.yahoo.container.di.ConfigRetriever.{BootstrapConfigs, ComponentsConfigs} +import com.yahoo.container.di.Container._ +import com.yahoo.container.di.componentgraph.core.ComponentNode.ComponentConstructorException +import com.yahoo.container.di.componentgraph.core.{ComponentGraph, ComponentNode, JerseyNode} +import com.yahoo.container.di.config.{RestApiContext, SubscriberFactory} import com.yahoo.container.{BundlesConfig, ComponentsConfig} +import com.yahoo.protect.Process +import com.yahoo.vespa.config.ConfigKey + +import scala.collection.JavaConversions._ +import scala.math.max +import scala.concurrent.duration._ +import scala.language.postfixOps /** @@ -43,27 +49,67 @@ class Container( def deconstructObsoleteComponents(oldGraph: ComponentGraph, newGraph: ComponentGraph) { val oldComponents = new IdentityHashMap[AnyRef, AnyRef]() - oldGraph.allComponentsAndProviders foreach(oldComponents.put(_, null)) - newGraph.allComponentsAndProviders foreach(oldComponents.remove(_)) - oldComponents.keySet foreach(componentDeconstructor.deconstruct(_)) + oldGraph.allComponentsAndProviders foreach (oldComponents.put(_, null)) + newGraph.allComponentsAndProviders foreach (oldComponents.remove(_)) + oldComponents.keySet foreach (componentDeconstructor.deconstruct(_)) } try { - //TODO: wrap user exceptions. val newGraph = createNewGraph(oldGraph, fallbackInjector) newGraph.reuseNodes(oldGraph) constructComponents(newGraph) deconstructObsoleteComponents(oldGraph, newGraph) newGraph } catch { - case e : Throwable => - invalidateGeneration() - throw e + case userException: ComponentConstructorException => + invalidateGeneration(oldGraph.generation, userException) + // TODO: Wrap userException in an Error when generation==0 (+ unit test that Error is thrown) + throw userException + case t: Throwable => + invalidateGeneration(oldGraph.generation, t) + throw t } } - private def invalidateGeneration() { + private def invalidateGeneration(generation: Long, cause: Throwable) { + val maxWaitToExit = 60 seconds + + def newGraphErrorMessage(generation: Long, cause: Throwable): String = { + val failedFirstMessage = "Failed to set up first component graph" + val failedNewMessage = "Failed to set up new component graph" + val constructMessage = "due to error when constructing one of the components" + val exitMessage = s"Exiting within $maxWaitToExit." + val retainMessage = "Retaining previous component generation." + generation match { + case 0 => + cause match { + case _: ComponentConstructorException => s"$failedFirstMessage $constructMessage. $exitMessage" + case _ => s"$failedFirstMessage. $exitMessage" + } + case _ => + cause match { + case _: ComponentConstructorException => s"$failedNewMessage $constructMessage. $retainMessage" + case _ => s"$failedNewMessage. $retainMessage" + } + } + } + + // TODO: move to ConfiguredApplication + def logAndDie(message: String, cause: Throwable): Unit = { + log.log(Level.SEVERE, message, cause) + try { + Thread.sleep((new Random(System.nanoTime).nextDouble * maxWaitToExit.toMillis).toLong) + } catch { + case _: InterruptedException => // Do nothing + } + Process.logAndDie("Exited for reason (repeated from above):", cause) + } + leastGeneration = max(configurer.getComponentsGeneration, configurer.getBootstrapGeneration) + 1 + cause match { + case _: InterruptedException | _: ConfigInterruptedException => // Normal during shutdown, do not log anything. + case _ => log.log(Level.WARNING, newGraphErrorMessage(generation, cause), cause) + } } final def createNewGraph(graph: ComponentGraph = new ComponentGraph, @@ -101,9 +147,8 @@ class Container( configurer.getComponentsGeneration } - private def createAndConfigureComponentsGraph[T]( - componentsConfigs: Map[ConfigKeyT, ConfigInstance], - fallbackInjector: Injector): ComponentGraph = { + private def createAndConfigureComponentsGraph[T](componentsConfigs: Map[ConfigKeyT, ConfigInstance], + fallbackInjector: Injector): ComponentGraph = { val componentGraph = createComponentsGraph(componentsConfigs, getComponentsGeneration, fallbackInjector) componentGraph.setAvailableConfigs(componentsConfigs) diff --git a/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/ComponentNode.scala b/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/ComponentNode.scala index 16856930e54..242d4c97a79 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/ComponentNode.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/ComponentNode.scala @@ -1,21 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di.componentgraph.core -import java.lang.reflect.{Modifier, ParameterizedType, Constructor, Type, InvocationTargetException} +import java.lang.reflect.{Constructor, InvocationTargetException, Modifier, ParameterizedType, Type} import java.util.logging.Logger -import com.google.inject.Inject +import com.google.inject.Inject +import com.yahoo.component.{AbstractComponent, ComponentId} import com.yahoo.config.ConfigInstance -import com.yahoo.vespa.config.ConfigKey -import com.yahoo.component.{ComponentId, AbstractComponent} -import com.yahoo.container.di.{ConfigKeyT, JavaAnnotation, createKey, makeClassCovariant, removeStackTrace, preserveStackTrace} import com.yahoo.container.di.componentgraph.Provider +import com.yahoo.container.di.componentgraph.core.ComponentNode._ +import com.yahoo.container.di.componentgraph.core.Node.equalEdges +import com.yahoo.container.di.{ConfigKeyT, JavaAnnotation, createKey, makeClassCovariant, preserveStackTrace, removeStackTrace} +import com.yahoo.vespa.config.ConfigKey -import Node.equalEdges -import ComponentNode._ -import java.lang.IllegalStateException -import scala.Some -import scala.Array +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future, TimeoutException} +import scala.language.postfixOps /** * @author tonytv @@ -95,23 +96,23 @@ class ComponentNode(componentId: ComponentId, } val instance = - try { - constructor.newInstance(actualArguments: _*) - } catch { - case e: InvocationTargetException => - throw removeStackTrace(constructThrowable(cutStackTraceAtConstructor(e.getCause), s"Error constructing $idAndType")) - } + try { + constructor.newInstance(actualArguments: _*) + } catch { + case e: InvocationTargetException => + throw removeStackTrace(ErrorOrComponentConstructorException(cutStackTraceAtConstructor(e.getCause), s"Error constructing $idAndType")) + } initId(instance) } - - private def constructThrowable(cause: Throwable, message: String) : Throwable = { + + private def ErrorOrComponentConstructorException(cause: Throwable, message: String) : Throwable = { if (cause != null && cause.isInstanceOf[Error]) // don't convert Errors to RuntimeExceptions new Error(message, cause) else - new RuntimeException(message, cause) + new ComponentConstructorException(message, cause) } - + private def initId(component: AnyRef) = { def checkAndSetId(c: AbstractComponent) { if (c.hasInitializedId && c.getId != componentId ) @@ -180,7 +181,7 @@ object ComponentNode { publicConstructors filter {_.getAnnotation(classOf[Inject]) != null} match { case Array() => None case Array(single) => Some(single) - case _ => throwRuntimeExceptionRemoveStackTrace("Multiple constructors annotated with inject in class " + clazz.getName) + case _ => throwComponentConstructorException("Multiple constructors annotated with inject in class " + clazz.getName) } } @@ -188,7 +189,7 @@ object ComponentNode { def isConfigInstance(clazz: Class[_]) = classOf[ConfigInstance].isAssignableFrom(clazz) publicConstructors match { - case Array() => throwRuntimeExceptionRemoveStackTrace("No public constructors in class " + clazz.getName) + case Array() => throwComponentConstructorException("No public constructors in class " + clazz.getName) case Array(single) => single case _ => log.warning("Multiple public constructors found in class %s, there should only be one. ".format(clazz.getName) + @@ -202,9 +203,12 @@ object ComponentNode { constructorAnnotatedWithInject getOrElse constructorWithMostConfigParameters } - private def throwRuntimeExceptionRemoveStackTrace(message: String) = - throw removeStackTrace(new RuntimeException(message)) + private def throwComponentConstructorException(message: String) = + throw removeStackTrace(new ComponentConstructorException(message)) + class ComponentConstructorException(message: String, cause: Throwable) extends RuntimeException(message, cause) { + def this(message: String) = this(message, null) + } def isAbstract(clazz: Class[_ <: AnyRef]) = Modifier.isAbstract(clazz.getModifiers) } diff --git a/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/Node.scala b/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/Node.scala index 9d30552b0aa..2ec3b77ed8a 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/Node.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/componentgraph/core/Node.scala @@ -72,7 +72,7 @@ abstract class Node(val componentId: ComponentId) { val className = instanceType.getName if (className == componentId.getName) s"'$componentId'" - else s"'$componentId of type '$className'" + else s"'$componentId' of type '$className'" } } diff --git a/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala b/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala index b64de80e39f..9246f7e22c3 100644 --- a/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala +++ b/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala @@ -1,30 +1,35 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.di -import com.yahoo.container.di.componentgraph.core.ComponentGraphTest.{SimpleComponent2, SimpleComponent} +import com.yahoo.container.di.componentgraph.core.ComponentGraphTest.{SimpleComponent, SimpleComponent2} import com.yahoo.container.di.componentgraph.Provider -import com.yahoo.container.di.componentgraph.core.{Node, ComponentGraph} -import org.junit.{Test, Before, After} +import com.yahoo.container.di.componentgraph.core.{ComponentGraph, ComponentNode, Node} +import org.junit.{After, Before, Ignore, Test} import org.junit.Assert._ import org.hamcrest.CoreMatchers._ import com.yahoo.config.test.TestConfig import com.yahoo.component.AbstractComponent import ContainerTest._ + import scala.collection.JavaConversions import com.yahoo.config.di.IntConfig -import scala.concurrent.{future, Await} + +import scala.concurrent.{Await, Future, future} import scala.concurrent.duration._ import scala.concurrent.ExecutionContext.Implicits.global import scala.util.Try import com.yahoo.container.di.config.RestApiContext import com.yahoo.container.bundle.MockBundle +import com.yahoo.container.di.componentgraph.core.ComponentNode.ComponentConstructorException + +import scala.language.postfixOps /** * @author tonytv * @author gjoranv */ class ContainerTest { - var dirConfigSource: DirConfigSource = null + var dirConfigSource: DirConfigSource = _ @Before def setup() { dirConfigSource = new DirConfigSource("ContainerTest-") @@ -88,8 +93,8 @@ class ContainerTest { container.reloadConfig(2) val newGraph = container.runOnce(graph) - assertThat(ComponentGraph.getNode(newGraph, "id1"), notNullValue(classOf[Node])); - assertThat(ComponentGraph.getNode(newGraph, "id2"), notNullValue(classOf[Node])); + assertThat(ComponentGraph.getNode(newGraph, "id1"), notNullValue(classOf[Node])) + assertThat(ComponentGraph.getNode(newGraph, "id2"), notNullValue(classOf[Node])) container.shutdownConfigurer() } @@ -113,8 +118,22 @@ class ContainerTest { assertTrue(componentToDestruct.deconstructed) } + @Ignore // because logAndDie is impossible(?) to verify programmatically + @Test + def manually_verify_what_happens_when_first_graph_contains_component_that_throws_exception_in_ctor() { + writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionInConstructor]) + val container = newContainer(dirConfigSource) + var currentGraph: ComponentGraph = null + try { + currentGraph = container.runOnce() + fail("Expected to log and die.") + } catch { + case _: Throwable => fail("Expected to log and die") + } + } + @Test - def previous_graph_is_retained_when_new_graph_throws_exception() { + def previous_graph_is_retained_when_new_graph_contains_component_that_throws_exception_in_ctor() { val simpleComponentEntry = ComponentEntry("simpleComponent", classOf[SimpleComponent]) writeBootstrapConfigs(Array(simpleComponentEntry)) @@ -123,26 +142,53 @@ class ContainerTest { val simpleComponent = currentGraph.getInstance(classOf[SimpleComponent]) - writeBootstrapConfigs("thrower", classOf[ComponentThrowingException]) - dirConfigSource.writeConfig("test", """stringVal "myString" """) + writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionInConstructor]) container.reloadConfig(2) try { currentGraph = container.runOnce(currentGraph) fail("Expected exception") } catch { - case e: Exception => e.printStackTrace() + case _: ComponentConstructorException => // Expected, do nothing + case _: Throwable => fail("Expected ComponentConstructorException") } + assertEquals(1, currentGraph.generation) + // Also verify that next reconfig is successful val componentTakingConfigEntry = ComponentEntry("componentTakingConfig", classOf[ComponentTakingConfig]) + dirConfigSource.writeConfig("test", """stringVal "myString" """) writeBootstrapConfigs(Array(simpleComponentEntry, componentTakingConfigEntry)) container.reloadConfig(3) currentGraph = container.runOnce(currentGraph) + assertEquals(3, currentGraph.generation) assertSame(simpleComponent, currentGraph.getInstance(classOf[SimpleComponent])) assertNotNull(currentGraph.getInstance(classOf[ComponentTakingConfig])) } @Test + def previous_graph_is_retained_when_new_graph_throws_exception_for_missing_config() { + val simpleComponentEntry = ComponentEntry("simpleComponent", classOf[SimpleComponent]) + + writeBootstrapConfigs(Array(simpleComponentEntry)) + val container = newContainer(dirConfigSource) + var currentGraph = container.runOnce() + + val simpleComponent = currentGraph.getInstance(classOf[SimpleComponent]) + + writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionForMissingConfig]) + dirConfigSource.writeConfig("test", """stringVal "myString" """) + container.reloadConfig(2) + try { + currentGraph = container.runOnce(currentGraph) + fail("Expected exception") + } catch { + case _: IllegalArgumentException => // Expected, do nothing + case _: Throwable => fail("Expected IllegalArgumentException") + } + assertEquals(1, currentGraph.generation) + } + + @Test def runOnce_hangs_waiting_for_valid_config_after_invalid_config() { dirConfigSource.writeConfig("test", """stringVal "original" """) writeBootstrapConfigs("myId", classOf[ComponentTakingConfig]) @@ -150,7 +196,7 @@ class ContainerTest { val container = newContainer(dirConfigSource) var currentGraph = container.runOnce() - writeBootstrapConfigs("thrower", classOf[ComponentThrowingException]) + writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionForMissingConfig]) container.reloadConfig(2) try { @@ -160,7 +206,7 @@ class ContainerTest { case e: Exception => } - val newGraph = future { + val newGraph = Future { currentGraph = container.runOnce(currentGraph) currentGraph } @@ -216,7 +262,7 @@ class ContainerTest { val anotherComponentClass = classOf[SimpleComponent2] val anotherComponentId = "anotherComponent" - val componentsConfig = + val componentsConfig: String = ComponentEntry(injectedComponentId, injectedClass).asConfig(0) + "\n" + ComponentEntry(anotherComponentId, anotherComponentClass).asConfig(1) + "\n" + ComponentEntry("restApiContext", restApiClass).asConfig(2) + "\n" + @@ -240,7 +286,7 @@ class ContainerTest { } case class ComponentEntry(componentId: String, classId: Class[_]) { - def asConfig(position: Int) = { + def asConfig(position: Int): String = { <config> |components[{position}].id "{componentId}" |components[{position}].classId "{classId.getName}" @@ -307,7 +353,7 @@ object ContainerTest { def get() = instance def deconstruct() { - require(instance.deconstructed == false) + require(! instance.deconstructed) instance.deconstructed = true } } @@ -316,8 +362,12 @@ object ContainerTest { require(config != null) } - class ComponentThrowingException(config:IntConfig) extends AbstractComponent { - throw new RuntimeException("This component can never be created") + class ComponentThrowingExceptionInConstructor() { + throw new RuntimeException("This component fails upon construction.") + } + + class ComponentThrowingExceptionForMissingConfig(intConfig: IntConfig) extends AbstractComponent { + fail("This component should never be created. Only used for tests where 'int' config is missing.") } class DestructableComponent extends AbstractComponent { @@ -346,5 +396,5 @@ object ContainerTest { componentGraph.getInstance(classOf[ComponentTakingConfig]) } - def convertMap[K, V](map: java.util.Map[K, V]) = JavaConversions.mapAsScalaMap(map).toMap + def convertMap[K, V](map: java.util.Map[K, V]): Map[K, V] = JavaConversions.mapAsScalaMap(map).toMap } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index e602813c263..c1548775bf7 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -287,7 +287,7 @@ public final class ConfiguredApplication implements Application { public void run() { com.yahoo.protect.Process.logAndDie( "Timed out waiting for application shutdown. Please check that all your request handlers " + - "drain their request content channels."); + "drain their request content channels.", true); } }, delayMillis, TimeUnit.MILLISECONDS); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 1c6e85a177a..4d71ae913d7 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -1,16 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import java.util.Timer; -import java.util.TimerTask; -import java.nio.file.DirectoryStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.Files; - import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.statistics.ActiveContainerStatistics; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Timer; +import java.util.TimerTask; /** * Uses a timer to emit metrics @@ -21,27 +22,29 @@ import com.yahoo.jdisc.Metric; */ public class MetricUpdater extends AbstractComponent { - private static final String FREE_MEMORY_BYTES = "freeMemoryBytes"; - private static final String USED_MEMORY_BYTES = "usedMemoryBytes"; - private static final String TOTAL_MEMORY_BYTES = "totalMemoryBytes"; - private static final String MANHATTAN_FREE_MEMORY_BYTES = "mem.heap.free"; - private static final String MANHATTAN_USED_MEMORY_BYTES = "mem.heap.used"; - private static final String MANHATTAN_TOTAL_MEMORY_BYTES = "mem.heap.total"; + @Deprecated private static final String DEPRECATED_FREE_MEMORY_BYTES = "freeMemoryBytes"; + @Deprecated private static final String DEPRECATED_USED_MEMORY_BYTES = "usedMemoryBytes"; + @Deprecated private static final String DEPRECATED_TOTAL_MEMORY_BYTES = "totalMemoryBytes"; + private static final String FREE_MEMORY_BYTES = "mem.heap.free"; + private static final String USED_MEMORY_BYTES = "mem.heap.used"; + private static final String TOTAL_MEMORY_BYTES = "mem.heap.total"; private static final String MEMORY_MAPPINGS_COUNT = "jdisc.memory_mappings"; private static final String OPEN_FILE_DESCRIPTORS = "jdisc.open_file_descriptors"; private final Metric metric; + private final ActiveContainerStatistics activeContainerStatistics; private final Timer timer = new Timer(); long freeMemory = -1; long totalMemory = -1; @Inject - public MetricUpdater(Metric metric) { - this(metric, 10*1000); + public MetricUpdater(Metric metric, ActiveContainerStatistics activeContainerStatistics) { + this(metric, activeContainerStatistics, 10*1000); } - public MetricUpdater(Metric metric, long delayMillis) { + public MetricUpdater(Metric metric, ActiveContainerStatistics activeContainerStatistics, long delayMillis) { this.metric = metric; + this.activeContainerStatistics = activeContainerStatistics; timer.schedule(new UpdaterTask(), delayMillis, delayMillis); } @@ -55,19 +58,21 @@ public class MetricUpdater extends AbstractComponent { long getTotalMemory() { return totalMemory; } private class UpdaterTask extends TimerTask { + @SuppressWarnings("deprecation") @Override public void run() { freeMemory = Runtime.getRuntime().freeMemory(); totalMemory = Runtime.getRuntime().totalMemory(); long usedMemory = totalMemory - freeMemory; + metric.set(DEPRECATED_FREE_MEMORY_BYTES, freeMemory, null); + metric.set(DEPRECATED_USED_MEMORY_BYTES, usedMemory, null); + metric.set(DEPRECATED_TOTAL_MEMORY_BYTES, totalMemory, null); metric.set(FREE_MEMORY_BYTES, freeMemory, null); metric.set(USED_MEMORY_BYTES, usedMemory, null); metric.set(TOTAL_MEMORY_BYTES, totalMemory, null); - metric.set(MANHATTAN_FREE_MEMORY_BYTES, freeMemory, null); - metric.set(MANHATTAN_USED_MEMORY_BYTES, usedMemory, null); - metric.set(MANHATTAN_TOTAL_MEMORY_BYTES, totalMemory, null); metric.set(MEMORY_MAPPINGS_COUNT, count_mappings(), null); metric.set(OPEN_FILE_DESCRIPTORS, count_open_files(), null); + activeContainerStatistics.emitMetrics(metric); } // Note: Linux-specific diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index d76be872d63..dd79e0dd054 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -247,13 +247,6 @@ maybe_use_jsvc () { fi } -maybe_run_extensions() { - vespa_extensions_script="${VESPA_HOME}libexec/vespa/jdisc-extensions.sh" - if [ -f "${vespa_extensions_script}" ]; then - . "${vespa_extensions_script}" - fi -} - getconfig configure_memory configure_gcopts @@ -262,7 +255,6 @@ configure_classpath configure_numactl configure_preload maybe_use_jsvc -maybe_run_extensions exec $numactlcmd $envcmd java \ -Dconfig.id="${VESPA_CONFIG_ID}" \ diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index 1067b572644..1b88b5217bc 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import static org.junit.Assert.assertTrue; - +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.jdisc.statistics.ActiveContainerStatistics; import org.junit.Test; import org.mockito.Mockito; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.application.MetricConsumer; +import static org.junit.Assert.assertTrue; public class MetricUpdaterTest { @@ -17,7 +17,7 @@ public class MetricUpdaterTest { MetricProvider provider = MetricProviders.newInstance(consumer); Metric metric = provider.get(); - MetricUpdater updater = new MetricUpdater(metric, 10); + MetricUpdater updater = new MetricUpdater(metric, Mockito.mock(ActiveContainerStatistics.class), 10); long start = System.currentTimeMillis(); boolean updated = false; while (System.currentTimeMillis() - start < 60000 && !updated) { diff --git a/container-messagebus/OWNERS b/container-messagebus/OWNERS index 123437e2758..12b533ec610 100644 --- a/container-messagebus/OWNERS +++ b/container-messagebus/OWNERS @@ -1 +1 @@ -dybis +havardpe diff --git a/container-search/.gitignore b/container-search/.gitignore index e4687268e83..7f4ebc3a7c6 100644 --- a/container-search/.gitignore +++ b/container-search/.gitignore @@ -1,54 +1,31 @@ -prelude.iml -accessLog.log +.classpath +.project +.settings /.cache +/.classpath /.emacs.desktop +/.nbintdb +/.project +/.settings /.version -/libexec -/testLogs -/build /QueryAccessLog* -/tmp -/.classpath -/.project -/.nbintdb +/accessLog.log +/build +/bundles +/libexec /nbproject -/prelude.iws +/pom.xml.build /prelude.iml /prelude.ipr -/bundles -/testLogFileG.txt -/testlogsG -/.settings -/test_yapache_access_log -/target +/prelude.iws /staging -/accessLog.log +/target /temp - -target - -staging - -temp - -tmp - -testlogsG - -.classpath - -.project - -QueryAccessLog.* - -test_yapache_access_log - -testLogFileG.txt - -.settings - -bundles -null.log - +/testLogFileG.txt +/testLogs +/test_yapache_access_log +/testlogsG /testng.out.log -/pom.xml.build +/tmp +null.log +tmp diff --git a/container-search/src/main/java/com/yahoo/.gitignore b/container-search/src/main/java/com/yahoo/.gitignore deleted file mode 100644 index 3fcbf90a554..00000000000 --- a/container-search/src/main/java/com/yahoo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/ - diff --git a/container-search/src/main/java/com/yahoo/component/chain/dependencies/.gitignore b/container-search/src/main/java/com/yahoo/component/chain/dependencies/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/component/chain/dependencies/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/component/provider/.gitignore b/container-search/src/main/java/com/yahoo/component/provider/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/component/provider/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/config/.gitignore b/container-search/src/main/java/com/yahoo/config/.gitignore deleted file mode 100644 index 28c283f2219..00000000000 --- a/container-search/src/main/java/com/yahoo/config/.gitignore +++ /dev/null @@ -1,34 +0,0 @@ -/QueryProfilesConfig.java -/QrBinaryCacheRegionConfig.java -/ClusterConfig.java -/QrSearchersConfig.java -/PhysicalmappingConfig.java -/QrMonitorConfig.java -/BundlesConfig.java -/SummaryConfig.java -/RankProfilesConfig.java -/FieldInfoConfig.java -/QrStartConfig.java -/StringConfig.java -/QrFileserverConfig.java -/QrBinaryCacheConfig.java -/SemanticRulesConfig.java -/FederationConfig.java -/QrConfig.java -/IndexInfoConfig.java -/QueryplannerConfig.java -/AttributesConfig.java -/QrLoggingConfig.java -/ProviderConfig.java -/IntConfig.java -/SearchChainsConfig.java -/PageTemplatesConfig.java -/HandlersConfig.java -/QrTemplatesConfig.java -/TilingSearcherConfig.java -/TimingSearcherConfig.java -/RewritesConfig.java -/UnmarshallingConfig.java -/SpecialtokensConfig.java -/QrQuotetableConfig.java -/ComponentsConfig.java diff --git a/container-search/src/main/java/com/yahoo/container/config/webapp/.gitignore b/container-search/src/main/java/com/yahoo/container/config/webapp/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/config/webapp/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/handler/config/.gitignore b/container-search/src/main/java/com/yahoo/container/handler/config/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/handler/config/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/handler/observability/.gitignore b/container-search/src/main/java/com/yahoo/container/handler/observability/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/handler/observability/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/http/.gitignore b/container-search/src/main/java/com/yahoo/container/http/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/http/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/jrt/.gitignore b/container-search/src/main/java/com/yahoo/container/jrt/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/jrt/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/logging/.gitignore b/container-search/src/main/java/com/yahoo/container/logging/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/logging/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/osgi/.gitignore b/container-search/src/main/java/com/yahoo/container/osgi/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/osgi/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/container/protect/.gitignore b/container-search/src/main/java/com/yahoo/container/protect/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/container/protect/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 1b44d60cce9..e210dab5299 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -276,11 +276,11 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { List<Result> parts = new ArrayList<>(); TinyIdentitySet<Query> queryMap = new TinyIdentitySet<>(4); - for (Iterator<Hit> itr = hitIterator(result); itr.hasNext(); ) { - Hit hit = itr.next(); + for (Iterator<Hit> i = hitIterator(result); i.hasNext(); ) { + Hit hit = i.next(); if (hit instanceof FastHit) { FastHit fastHit = (FastHit) hit; - if (!fastHit.isFilled(summaryClass)) { + if ( ! fastHit.isFilled(summaryClass)) { Query q = fastHit.getQuery(); if (q == null) { q = result.hits().getQuery(); // fallback for untagged hits diff --git a/container-search/src/main/java/com/yahoo/prelude/grouping/legacy/.gitignore b/container-search/src/main/java/com/yahoo/prelude/grouping/legacy/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/grouping/legacy/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 7f88cfd2a9b..224fa8e92fc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -87,6 +87,7 @@ public class TermType { return name.hashCode(); } - public @Override String toString() { return "term type '" + name + "'"; } + @Override + public String toString() { return "term type '" + name + "'"; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java index a89231b30ae..29b4bce1046 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java @@ -93,10 +93,12 @@ public class WordItem extends TermItem { * Returns this word as it should be used in executing the query. * This is usually (but not always) a normalized and stemmed form */ - public @Override String stringValue() { return word; } + @Override + public String stringValue() { return word; } /** Same as #setWord */ - public @Override void setValue(String value) { setWord(value); } + @Override + public void setValue(String value) { setWord(value); } /** * Get the word exactly as received in the request. diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 11206fbb1c1..c7e3412facc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -267,9 +267,8 @@ public abstract class AbstractParser implements CustomParser { * @return The normalized string. */ protected String normalize(String input) { - if (input == null || input.length() == 0) { - return input; - } + if (input == null || input.length() == 0) return input; + return environment.getLinguistics().getNormalizer().normalize(input); } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java index 86fdee2bad9..12a904a29e2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java @@ -389,7 +389,8 @@ public class PhraseMatcher { public int getLength() { return 1; } - public @Override TermItem getItem(int index) { + @Override + public TermItem getItem(int index) { if (index!=0) throw new IndexOutOfBoundsException("No word at " + index + " in " + this); return item; } @@ -455,7 +456,8 @@ public class PhraseMatcher { return startIndex==0 && length==owner.getItemCount(); } - public @Override TermItem getItem(int index) { + @Override + public TermItem getItem(int index) { adjustIfBackingChanged(); return (TermItem)owner.getItem(startIndex+index); } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java index ac2196bb9f5..2baa4698145 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java @@ -26,7 +26,8 @@ public class MultipleResultsSearcher extends Searcher { private static final CompoundName maxTimesRetrieveHeterogeneousHitsName=new CompoundName(propertyPrefix + "maxTimesRetrieveHeterogeneousHits"); private static final CompoundName numHits=new CompoundName(propertyPrefix + "numHits"); - public @Override Result search(Query query, Execution e) { + @Override + public Result search(Query query, Execution e) { try { Parameters parameters = new Parameters(query); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 3b2e782e1bb..3779202b5d0 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -182,9 +182,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** The time this query was created */ private long startTime; - /** Error conditions stemming from the query itself */ - private List<ErrorMessage> errors = new ArrayList<>(0); - //---------------- Static property handling ------------------------------------ public static final CompoundName OFFSET = new CompoundName("offset"); @@ -939,9 +936,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { clone.presentation = (Presentation) presentation.clone(); clone.context = getContext(true).cloneFor(clone); - if (errors != null) - clone.errors = new ArrayList<>(errors); - // Correct the Query instance in properties clone.properties().setParentQuery(clone); assert (clone.properties().getParentQuery() == clone); diff --git a/container-search/src/main/java/com/yahoo/search/Searcher.java b/container-search/src/main/java/com/yahoo/search/Searcher.java index 95b4f92ca56..cf41b01144e 100644 --- a/container-search/src/main/java/com/yahoo/search/Searcher.java +++ b/container-search/src/main/java/com/yahoo/search/Searcher.java @@ -153,21 +153,29 @@ public abstract class Searcher extends Processor { /** * Fills the result if it is not already filled for the given summary class. * See the fill method. - **/ + */ public final void ensureFilled(Result result, String summaryClass, Execution execution) { if (summaryClass == null) summaryClass = result.getQuery().getPresentation().getSummary(); - if (!result.isFilled(summaryClass)) { + if ( ! result.isFilled(summaryClass)) { fill(result, summaryClass, execution); } + else { + int fillRejectTraceAt = 3; + if (result.getQuery().getTraceLevel() >= fillRejectTraceAt) + result.getQuery().trace("Ignoring fill(" + summaryClass + "): " + + ( result.hits().getFilled() == null ? "Hits are unfillable" : "Hits already filled" ) + + ": result.hits().getFilled()=" + result.hits().getFilled(), fillRejectTraceAt); + } } /** Returns a logger unique for the instance subclass */ protected Logger getLogger() { return logger; } /** Returns "searcher 'getId()'" */ - public @Override String toString() { + @Override + public String toString() { return "searcher '" + getIdString() + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/app/.gitignore b/container-search/src/main/java/com/yahoo/search/app/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/search/app/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index f660f39656d..527d44b16ca 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -313,7 +313,8 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod getLogger().log(level, sb.toString()); } - public @Override void deconstruct() { + @Override + public void deconstruct() { super.deconstruct(); monitor.shutdown(); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java index 66bb678e983..594481b667d 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java @@ -29,7 +29,8 @@ public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { } /** Forwards to the next in chain fill(result,summaryName) */ - public @Override void fill(Result result,String summaryName, Execution execution,Connection connection) { + @Override + public void fill(Result result,String summaryName, Execution execution,Connection connection) { execution.fill(result,summaryName); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java index 25253f768bd..8e3225703f6 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java @@ -62,7 +62,8 @@ public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearche * Override this to provider multi-phase result filling towards a backend. * This default implementation does nothing. */ - public @Override void fill(Result result,String summaryName, Execution execution,Connection connection) { + @Override + public void fill(Result result,String summaryName, Execution execution,Connection connection) { } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java index abc87a92ab1..b8ddf0cb1ce 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java @@ -10,7 +10,7 @@ import com.yahoo.search.result.Hit; * contains one or more {@link Hit hits} itself, making this the parent of leaf nodes in the hierarchy of grouping * results. Use the {@link GroupingRequest#getResultGroup(Result)} to retrieve grouping results.</p> * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class HitList extends AbstractList { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 70942fa2553..0f1322c08d0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.objects.ObjectPredicate; * transformation from the abstract request to Vespa grouping expressions (using {@link RequestBuilder}), and the * corresponding transformation of results (using {@link ResultBuilder}). * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ @After({ GroupingValidator.GROUPING_VALIDATED, "com.yahoo.search.querytransform.WandSearcher", @@ -83,9 +83,9 @@ public class GroupingExecutor extends Searcher { // Convert requests to Vespa style grouping. Map<Integer, Grouping> groupingMap = new HashMap<>(); - List<RequestContext> ctxList = new LinkedList<>(); + List<RequestContext> requestContextList = new LinkedList<>(); for (GroupingRequest grpRequest : reqList) { - ctxList.add(convertRequest(query, grpRequest, groupingMap)); + requestContextList.add(convertRequest(query, grpRequest, groupingMap)); } if (groupingMap.isEmpty()) { return execution.search(query); @@ -96,10 +96,10 @@ public class GroupingExecutor extends Searcher { // Convert Vespa style results to hits. HitConverter hitConverter = new HitConverter(this, query); - for (RequestContext ctx : ctxList) { - RootGroup grp = convertResult(ctx, groupingMap, hitConverter); - ctx.request.setResultGroup(grp); - result.hits().add(grp); + for (RequestContext context : requestContextList) { + RootGroup group = convertResult(context, groupingMap, hitConverter); + context.request.setResultGroup(group); + result.hits().add(group); } return result; } @@ -110,11 +110,21 @@ public class GroupingExecutor extends Searcher { for (Iterator<Hit> it = result.hits().unorderedDeepIterator(); it.hasNext(); ) { Hit hit = it.next(); Object metaData = hit.getSearcherSpecificMetaData(this); - String hitSummary = (metaData instanceof String) ? (String)metaData : summaryClass; - Result summaryResult = summaryMap.get(hitSummary); + if (metaData != null && metaData instanceof String) { + // Use the summary class specified by grouping, set in HitConverter, for the first fill request + // after grouping. This assumes the first fill request is using the default summary class, + // which may be a fragile assumption. But currently we cannot do better because the difference + // between explicit and implicit summary class in fill is erased by the Execution. + // + // We reset the summary class here such that following fill calls will execute with the + // summary class they specify + summaryClass = (String) metaData; + hit.setSearcherSpecificMetaData(this, null); + } + Result summaryResult = summaryMap.get(summaryClass); if (summaryResult == null) { summaryResult = new Result(result.getQuery()); - summaryMap.put(hitSummary, summaryResult); + summaryMap.put(summaryClass, summaryResult); } summaryResult.hits().add(hit); } @@ -169,18 +179,18 @@ public class GroupingExecutor extends Searcher { /** * Converts the results of the given request context into a single {@link Group}. * - * @param requestCtx The context that identifies the results to convert. + * @param requestContext The context that identifies the results to convert. * @param groupingMap The map of all {@link Grouping} objects available. * @param hitConverter The converter to use for {@link Hit} conversion. * @return The corresponding root RootGroup. */ - private RootGroup convertResult(RequestContext requestCtx, Map<Integer, Grouping> groupingMap, + private RootGroup convertResult(RequestContext requestContext, Map<Integer, Grouping> groupingMap, HitConverter hitConverter) { ResultBuilder builder = new ResultBuilder(); builder.setHitConverter(hitConverter); - builder.setTransform(requestCtx.transform); - builder.setRequestId(requestCtx.request.getRequestId()); - for (Integer grpId : requestCtx.idList) { + builder.setTransform(requestContext.transform); + builder.setRequestId(requestContext.request.getRequestId()); + for (Integer grpId : requestContext.idList) { builder.addGroupingResult(groupingMap.get(grpId)); } builder.build(); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java index 81ae100b84f..388f528b535 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java @@ -14,7 +14,7 @@ import com.yahoo.searchlib.aggregation.VdsHit; /** * Implementation of the {@link ResultBuilder.HitConverter} interface for {@link GroupingExecutor}. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ class HitConverter implements ResultBuilder.HitConverter { @@ -43,32 +43,32 @@ class HitConverter implements ResultBuilder.HitConverter { } } - private Hit convertFs4Hit(String summaryClass, FS4Hit grpHit) { - FastHit ret = new FastHit(); - ret.setRelevance(grpHit.getRank()); - ret.setGlobalId(grpHit.getGlobalId()); - ret.setPartId(grpHit.getPath(), 0); - ret.setDistributionKey(grpHit.getDistributionKey()); - ret.setFillable(); - ret.setSearcherSpecificMetaData(searcher, summaryClass); + private Hit convertFs4Hit(String summaryClass, FS4Hit groupHit) { + FastHit hit = new FastHit(); + hit.setRelevance(groupHit.getRank()); + hit.setGlobalId(groupHit.getGlobalId()); + hit.setPartId(groupHit.getPath(), 0); + hit.setDistributionKey(groupHit.getDistributionKey()); + hit.setFillable(); + hit.setSearcherSpecificMetaData(searcher, summaryClass); - Hit ctxHit = (Hit)grpHit.getContext(); + Hit ctxHit = (Hit)groupHit.getContext(); if (ctxHit == null) { throw new NullPointerException("Hit has no context."); } - ret.setSource(ctxHit.getSource()); - ret.setSourceNumber(ctxHit.getSourceNumber()); - ret.setQuery(ctxHit.getQuery()); + hit.setSource(ctxHit.getSource()); + hit.setSourceNumber(ctxHit.getSourceNumber()); + hit.setQuery(ctxHit.getQuery()); if (ctxHit instanceof GroupingListHit) { // in a live system the ctxHit can only by GroupingListHit, but because the code used Hit prior to version // 5.10 we need to check to avoid breaking existing unit tests -- both internally and with customers QueryPacketData queryPacketData = ((GroupingListHit)ctxHit).getQueryPacketData(); if (queryPacketData != null) { - ret.setQueryPacketData(queryPacketData); + hit.setQueryPacketData(queryPacketData); } } - return ret; + return hit; } private Hit convertVdsHit(String summaryClass, VdsHit grpHit) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index 6caffc0d043..9b3601d35cb 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -52,7 +52,7 @@ import java.util.Map; * This class implements the necessary logic to build a {@link RootGroup} from a list of {@link Grouping} objects. It is * used by the {@link GroupingExecutor}. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ class ResultBuilder { @@ -383,7 +383,7 @@ class ResultBuilder { * Defines a helper interface to convert Vespa style grouping hits into corresponding instances of {@link Hit}. It * is an interface to simplify testing. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public interface HitConverter { diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java b/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java index 25962090a69..bdf0caaac6d 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java @@ -25,15 +25,18 @@ public class Intent { /** Returns the id of this intent, never null */ public String getId() { return id; } - public @Override int hashCode() { return id.hashCode(); } + @Override + public int hashCode() { return id.hashCode(); } - public @Override boolean equals(Object other) { + @Override + public boolean equals(Object other) { if (other==this) return true; if ( ! (other instanceof Intent)) return false; return this.id.equals(((Intent)other).id); } /** Returns the id of this intent */ - public @Override String toString() { return id; } + @Override + public String toString() { return id; } } diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java index 113bad46312..345c041d212 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java @@ -22,7 +22,8 @@ public class IntentNode extends ParentNode<SourceNode> { public void setIntent(Intent intent) { this.intent=intent; } /** Returns intent:probability */ - public @Override String toString() { + @Override + public String toString() { return intent + ":" + getScore(); } diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java index d54025bb0c6..9101d977e16 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java @@ -28,7 +28,8 @@ public class InterpretationNode extends ParentNode<IntentNode> { } /** Returns the probability of the interpretation of this */ - public @Override double getScore() { + @Override + public double getScore() { return interpretation.getProbability(); } @@ -38,7 +39,8 @@ public class InterpretationNode extends ParentNode<IntentNode> { } /** Returns interpretations toString() */ - public @Override String toString() { + @Override + public String toString() { return interpretation.toString(); } diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/Source.java b/container-search/src/main/java/com/yahoo/search/intent/model/Source.java index f224612f23f..cd6b8cca28d 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/Source.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/Source.java @@ -23,15 +23,18 @@ public class Source { /** Returns the id of this source, never null */ public String getId() { return id; } - public @Override int hashCode() { return id.hashCode(); } + @Override + public int hashCode() { return id.hashCode(); } - public @Override boolean equals(Object other) { + @Override + public boolean equals(Object other) { if (other==this) return true; if ( ! (other instanceof Source)) return false; return this.id.equals(((Source)other).id); } /** Returns the id of this source */ - public @Override String toString() { return id; } + @Override + public String toString() { return id; } } diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java index 84c98b8c723..4fcf93809ef 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java @@ -34,7 +34,8 @@ public class SourceNode extends Node { } /** Returns source:appropriateness */ - public @Override String toString() { + @Override + public String toString() { return source + ":" + getScore(); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java index 8c421feae47..915b44ce5e0 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java @@ -49,7 +49,8 @@ public final class PageTemplate extends FreezableComponent implements PageElemen return Collections.unmodifiableSet(sourceVisitor.getSources()); } - public @Override void freeze() { + @Override + public void freeze() { if (isFrozen()) return; resolvePlaceholders(); section.freeze(); @@ -70,12 +71,14 @@ public final class PageTemplate extends FreezableComponent implements PageElemen } /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); section.accept(visitor); } - public @Override String toString() { + @Override + public String toString() { return "page template '" + getId() + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java index de19e8ab747..8dcfde01d20 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java @@ -21,7 +21,8 @@ class PlaceholderMappingVisitor extends PageTemplateVisitor { private Map<String, MapChoice> placeholderIdToChoice=new LinkedHashMap<>(); - public @Override void visit(MapChoice mapChoice) { + @Override + public void visit(MapChoice mapChoice) { List<String> placeholderIds=mapChoice.placeholderIds(); for (String placeholderId : placeholderIds) { MapChoice existingChoice=placeholderIdToChoice.put(placeholderId,mapChoice); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java index c464dd3dedc..8f1413c8c63 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java @@ -20,7 +20,8 @@ class PlaceholderReferenceCreatingVisitor extends PageTemplateVisitor { this.placeholderIdToChoice=placeholderIdToChoice; } - public @Override void visit(Placeholder placeholder) { + @Override + public void visit(Placeholder placeholder) { MapChoice choice=placeholderIdToChoice.get(placeholder.getId()); if (choice==null) throw new IllegalArgumentException(placeholder + " is not referenced by any choice"); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java index 5349d9f2aaa..2ae95dc97d1 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java @@ -19,7 +19,8 @@ class RelevanceComparator extends ChainableComparator { super(secondaryComparator); } - public @Override int compare(Hit h1,Hit h2) { + @Override + public int compare(Hit h1,Hit h2) { int relevanceComparison=h2.getRelevance().compareTo(h1.getRelevance()); if (relevanceComparison!=0) return relevanceComparison; diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java index 3ee3e3aeef0..6cfe8be50d3 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java @@ -59,7 +59,8 @@ public class Resolution { return resolution; } - public @Override String toString() { + @Override + public String toString() { return "a resolution of " + choiceResolutions.size() + " choices"; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java index 6415dad37ec..1bb46ac14a6 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java @@ -98,12 +98,14 @@ public abstract class Resolver extends AbstractComponent { this.resolution=resolution; } - public @Override void visit(Choice choice) { + @Override + public void visit(Choice choice) { if (choice.alternatives().size()<2) return; // No choice... resolve(choice,query,result,resolution); } - public @Override void visit(MapChoice choice) { + @Override + public void visit(MapChoice choice) { resolve(choice,query,result,resolution); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java index e0185554066..a0db7c74ecd 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java @@ -34,7 +34,8 @@ class SourceOrderComparator extends ChainableComparator { this.sourceOrder=sourceOrder; } - public @Override int compare(Hit h1,Hit h2) { + @Override + public int compare(Hit h1,Hit h2) { int primaryOrder=sourceOrderCompare(h1,h2); if (primaryOrder!=0) return primaryOrder; diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java index 4100ca5504c..f7df75bffc2 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java @@ -27,9 +27,11 @@ public class Layout { public String getName() { return name; } - public @Override int hashCode() { return name.hashCode(); } + @Override + public int hashCode() { return name.hashCode(); } - public @Override boolean equals(Object o) { + @Override + public boolean equals(Object o) { if (o==this) return true; if (! (o instanceof Layout)) return false; Layout other=(Layout)o; @@ -45,6 +47,7 @@ public class Layout { return new Layout(layout); } - public @Override String toString() { return "layout '" + name + "'"; } + @Override + public String toString() { return "layout '" + name + "'"; } } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java index ccfad1673aa..bff04df2ee8 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java @@ -54,7 +54,8 @@ public class MapChoice extends AbstractChoice { } /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); for (List<PageElement> valueEntry : values) for (PageElement value : valueEntry) diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java index 0eb98f329cb..79743194c74 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java @@ -26,14 +26,17 @@ public class Placeholder implements PageElement { public void setValueContainer(MapChoice valueContainer) { this.valueContainer=valueContainer; } - public @Override void freeze() {} + @Override + public void freeze() {} /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); } - public @Override String toString() { + @Override + public String toString() { return "source placeholder '" + id + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java index c3a46dbee8b..261df787b80 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java @@ -71,17 +71,20 @@ public final class Renderer extends FreezableClass implements PageElement { */ public Map<String,String> parameters() { return parameters; } - public @Override void freeze() { + @Override + public void freeze() { if (isFrozen()) return; super.freeze(); parameters = Collections.unmodifiableMap(parameters); } /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); } - public @Override String toString() { + @Override + public String toString() { return "renderer '" + name + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java index 67f67cb8538..b0f49eddc83 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java @@ -149,7 +149,8 @@ public class Section extends FreezableClass implements PageElement { this.min=min; } - public @Override void freeze() { + @Override + public void freeze() { if (isFrozen()) return; for (PageElement element : elements) @@ -163,13 +164,15 @@ public class Section extends FreezableClass implements PageElement { } /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); for (PageElement element : elements) element.accept(visitor); } - public @Override String toString() { + @Override + public String toString() { if (id==null || id.isEmpty()) return "a section"; return "section '" + id + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java index de1cadff5db..7877aae5ca8 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java @@ -75,7 +75,8 @@ public class Source extends FreezableClass implements PageElement { */ public Map<String,String> parameters() { return parameters; } - public @Override void freeze() { + @Override + public void freeze() { if (isFrozen()) return; for (PageElement element : renderers) { if (element instanceof Renderer) { @@ -103,13 +104,15 @@ public class Source extends FreezableClass implements PageElement { } /** Accepts a visitor to this structure */ - public @Override void accept(PageTemplateVisitor visitor) { + @Override + public void accept(PageTemplateVisitor visitor) { visitor.visit(this); for (PageElement renderer : renderers) renderer.accept(visitor); } - public @Override int hashCode() { + @Override + public int hashCode() { if (isFrozen()) return hashCode; int hashCode=name.hashCode(); int i=0; @@ -118,7 +121,8 @@ public class Source extends FreezableClass implements PageElement { return hashCode; } - public @Override boolean equals(Object other) { + @Override + public boolean equals(Object other) { if (other==this) return true; if (! (other instanceof Source)) return false; Source otherSource=(Source)other; @@ -130,7 +134,8 @@ public class Source extends FreezableClass implements PageElement { return true; } - public @Override String toString() { + @Override + public String toString() { return "source '" + name + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index cef3d46b947..2208db23c72 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -21,7 +21,7 @@ import java.util.Set; /** * Parameters deciding how the result of a query should be presented * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + * @author Arne Bergene Fossaa */ public class Presentation implements Cloneable { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java index ab8cd49047d..cedbd45e831 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java @@ -19,11 +19,13 @@ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { super(prefix); } - public @Override void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner) { + @Override + public void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner) { putValue(localName, value, values); } - public @Override void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { + @Override + public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) { putValue("", profile.getValue(), values); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java index a4bca752d18..54b702720b2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -77,7 +77,8 @@ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { } /** Freezes this, and all owned query profiles and query profile types */ - public @Override void freeze() { + @Override + public void freeze() { if (isFrozen()) return; queryProfileTypeRegistry.freeze(); for (QueryProfile queryProfile : allComponents()) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java index 05bf7055822..41b5331259d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java @@ -88,7 +88,8 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions * are <i>set</i>, regardless of what they are set <i>to</i>. */ - public @Override int compareTo(QueryProfileVariant other) { + @Override + public int compareTo(QueryProfileVariant other) { return this.dimensionValues.compareTo(other.dimensionValues); } @@ -150,7 +151,8 @@ public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVa } } - public @Override String toString() { + @Override + public String toString() { return "query profile variant for " + dimensionValues; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java index 705ba7d3d29..4512168a046 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java @@ -390,7 +390,8 @@ public class QueryProfileVariants implements Freezable, Cloneable { } } - public @Override FieldValues clone() { + @Override + public FieldValues clone() { try { if (frozen) return this; FieldValues clone=(FieldValues)super.clone(); @@ -444,7 +445,8 @@ public class QueryProfileVariants implements Freezable, Cloneable { * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions * are <i>set</i>, regardless of what they are set <i>to</i>. */ - public @Override int compareTo(FieldValue other) { + @Override + public int compareTo(FieldValue other) { return this.dimensionValues.compareTo(other.dimensionValues); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java index c6af9e53494..c1bb6c8829e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java @@ -30,11 +30,13 @@ final class SingleValueQueryProfileVisitor extends QueryProfileVisitor { this.allowQueryProfileResult=allowQueryProfileResult; } - public @Override String getLocalKey() { + @Override + public String getLocalKey() { return name.get(nameIndex); } - public @Override boolean enter(String name) { + @Override + public boolean enter(String name) { if (nameIndex+1<this.name.size()) { nameIndex++; enteringContent=true; @@ -45,16 +47,19 @@ final class SingleValueQueryProfileVisitor extends QueryProfileVisitor { return enteringContent; } - public @Override void leave(String name) { + @Override + public void leave(String name) { nameIndex--; } - public @Override void onValue(String key,Object value, DimensionBinding binding, QueryProfile owner) { + @Override + public void onValue(String key,Object value, DimensionBinding binding, QueryProfile owner) { if (nameIndex==name.size()-1) this.value=value; } - public @Override void onQueryProfile(QueryProfile profile,DimensionBinding binding, QueryProfile owner) { + @Override + public void onQueryProfile(QueryProfile profile,DimensionBinding binding, QueryProfile owner) { if (enteringContent) return; // still waiting for content if (allowQueryProfileResult) this.value = profile; @@ -62,14 +67,16 @@ final class SingleValueQueryProfileVisitor extends QueryProfileVisitor { this.value = profile.getValue(); } - public @Override boolean isDone() { + @Override + public boolean isDone() { return value!=null; } /** Returns the value found during visiting, or null if none */ public Object getResult() { return value; } - public @Override String toString() { + @Override + public String toString() { return "a single value visitor (hash " + hashCode() + ") with current value " + value; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java index d0d18d7ba85..e73dbedaab3 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java @@ -76,7 +76,8 @@ public class SubstituteString { } /** Returns this string in original (unsubstituted) form */ - public @Override String toString() { + @Override + public String toString() { return stringValue; } @@ -94,11 +95,13 @@ public class SubstituteString { this.value=value; } - public @Override String getValue(Map<String,String> context,Properties substitution) { + @Override + public String getValue(Map<String,String> context,Properties substitution) { return value; } - public @Override String toString() { + @Override + public String toString() { return value; } @@ -112,13 +115,15 @@ public class SubstituteString { this.propertyName=propertyName; } - public @Override String getValue(Map<String,String> context,Properties substitution) { + @Override + public String getValue(Map<String,String> context,Properties substitution) { Object value=substitution.get(propertyName,context,substitution); if (value==null) return ""; return String.valueOf(value); } - public @Override String toString() { + @Override + public String toString() { return "%{" + propertyName + "}"; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java index 76b3f78ac2f..887968dc64b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java @@ -20,15 +20,19 @@ public class PrimitiveFieldType extends FieldType { this.primitiveClass=primitiveClass; } - public @Override Class getValueClass() { return primitiveClass; } + @Override + public Class getValueClass() { return primitiveClass; } - public @Override String stringValue() { + @Override + public String stringValue() { return toLowerCase(primitiveClass.getSimpleName()); } - public @Override String toString() { return "field type " + stringValue(); } + @Override + public String toString() { return "field type " + stringValue(); } - public @Override String toInstanceDescription() { + @Override + public String toInstanceDescription() { return toLowerCase(primitiveClass.getSimpleName()); } @@ -37,7 +41,8 @@ public class PrimitiveFieldType extends FieldType { return convertFrom(object, (QueryProfileRegistry)null); } - public @Override Object convertFrom(Object object, QueryProfileRegistry registry) { + @Override + public Object convertFrom(Object object, QueryProfileRegistry registry) { if (primitiveClass == object.getClass()) return object; if (object.getClass() == String.class) return convertFromString((String)object); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java index df52e78c6ef..724931b6c41 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -38,17 +38,21 @@ public class QueryProfileFieldType extends FieldType { /** Returns the query profile type of this, or null if any type works */ public QueryProfileType getQueryProfileType() { return type; } - public @Override Class<?> getValueClass() { return QueryProfile.class; } + @Override + public Class<?> getValueClass() { return QueryProfile.class; } - public @Override String stringValue() { + @Override + public String stringValue() { return "query-profile" + (type!=null ? ":" + type.getId().getName() : ""); } - public @Override String toString() { + @Override + public String toString() { return "field type " + stringValue(); } - public @Override String toInstanceDescription() { + @Override + public String toInstanceDescription() { return "reference to a query profile" + (type!=null ? " of type '" + type.getId().getName() + "'" : ""); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java index 536e31ba26e..a6ed3427cca 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java @@ -41,17 +41,20 @@ public class PropertyAliases extends Properties { return (properName != null) ? properName : nameOrAlias; } - public @Override Map<String, Object> listProperties(CompoundName property,Map<String,String> context, + @Override + public Map<String, Object> listProperties(CompoundName property,Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.listProperties(unalias(property),context,substitution); } - public @Override Object get(CompoundName name,Map<String,String> context, + @Override + public Object get(CompoundName name,Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.get(unalias(name),context,substitution); } - public @Override void set(CompoundName name,Object value,Map<String,String> context) { + @Override + public void set(CompoundName name,Object value,Map<String,String> context) { super.set(unalias(name),value,context); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 76d94f3f0a2..456b0df3fe5 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -39,7 +39,8 @@ public class PropertyMap extends Properties { */ protected boolean shouldSet(CompoundName name,Object value) { return true; } - public @Override Object get(CompoundName name, Map<String,String> context, + @Override + public Object get(CompoundName name, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { if ( ! properties.containsKey(name)) return super.get(name,context,substitution); return properties.get(name); @@ -52,7 +53,8 @@ public class PropertyMap extends Properties { return properties; } - public @Override PropertyMap clone() { + @Override + public PropertyMap clone() { PropertyMap clone = (PropertyMap)super.clone(); clone.properties = new HashMap<>(); for (Map.Entry<CompoundName, Object> entry : this.properties.entrySet()) { diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java index bb76c1006f2..9c39b93c4ec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java @@ -258,7 +258,8 @@ public abstract class QueryRewriteSearcher extends Searcher { * - Pass to the next rewriter the query to be used * for dictionary retrieval<br> */ - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { RewriterUtils.log(logger, query, "Executing " + getRewriterName()); // Check if rewriter is properly initialized diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java index 589696c4e77..f7f9489174d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java @@ -43,7 +43,8 @@ public class SearchChainDispatcherSearcher extends Searcher { * param<br> * - Execute specified search chain if exist */ - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { RewriterUtils.log(logger, query, "Entering SearchChainDispatcherSearcher"); // Retrieve search chain specified by user through REWRITER_CHAIN diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore deleted file mode 100644 index add88bd6807..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/TokenMgrError.java -/Token.java -/SimpleCharStream.java -/ParserTokenManager.java -/ParserConstants.java -/ParseException.java -/Parser.java diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore b/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java index 5fe604dca15..75f22323095 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java @@ -12,7 +12,7 @@ import java.util.Set; * En error hit maintains a main error - the main error of the result. * * @author bratseth - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { @@ -56,15 +56,6 @@ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { } /** - * Insert the new "main" error at head of list, remove from the list if it - * already exists elsewhere. - */ - private void removeAndAddAtHead(ErrorMessage mainError) { - errors.remove(mainError); // avoid error duplication - errors.add(0, mainError); - } - - /** * This is basically a way of making a list simulate a set. */ private void removeAndAdd(ErrorMessage error) { diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 2cf1dba7efd..cfc7ba7bce7 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -420,7 +420,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * to look at it that way. * * Note that you might need to overload isFilled if you overload this one. - **/ + */ public Set<String> getFilled() { return unmodifiableFilled; } @@ -709,10 +709,12 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } } - // TODO: Move out? If not, delegate here from subclass /** + * @deprecated do not use + * * @return a field without bolding markup */ + @Deprecated // TODO: Remove on Vespa 7 public String getUnboldedField(String key, boolean escape) { Object p = getField(key); @@ -732,7 +734,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** - * set meta data describing how a given searcher should treat this hit. + * Set meta data describing how a given searcher should treat this hit. * It is currently recommended that the invoker == searcher. * <b>Internal. Do not use!</b> */ diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index e58c3dc847e..3ce808eb75c 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -620,7 +620,8 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< * Returns true if all the hits recursively contained in this * is cached */ - public @Override boolean isCached() { + @Override + public boolean isCached() { if (notCachedCount<1) return true; if (subgroupCount<1) return false; // No need to check below diff --git a/container-search/src/main/java/com/yahoo/search/result/Relevance.java b/container-search/src/main/java/com/yahoo/search/result/Relevance.java index df79b64585e..a6a83dc3c5b 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Relevance.java +++ b/container-search/src/main/java/com/yahoo/search/result/Relevance.java @@ -45,7 +45,8 @@ public class Relevance implements Comparable<Relevance> { /** * Returns the score value as a string */ - public @Override String toString() { + @Override + public String toString() { return DoubleFormatter.stringValue(score); } @@ -67,7 +68,8 @@ public class Relevance implements Comparable<Relevance> { } /** Compares relevancy values */ - public @Override boolean equals(Object object) { + @Override + public boolean equals(Object object) { if (object==this) return true; if (!(object instanceof Relevance)) { return false; } @@ -77,7 +79,8 @@ public class Relevance implements Comparable<Relevance> { } /** Returns a hash from the relevancy value */ - public @Override int hashCode() { + @Override + public int hashCode() { double hash=getScore()*335451367; // A largish prime if (hash>-1 && hash<1) hash=1/hash; return (int) hash; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index c9176b208a4..cde03c9069d 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -516,9 +516,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { final int traceDependencies = 6; Query query = (Query) request; if (query.getTraceLevel() >= traceDependencies) { - query.trace(new StringBuilder().append(processor.getId()) - .append(" ").append(processor.getDependencies().toString()) - .toString(), traceDependencies); + query.trace(processor.getId() + " " + processor.getDependencies(), traceDependencies); } } @@ -564,21 +562,35 @@ public class Execution extends com.yahoo.processing.execution.Execution { } /** Calls fill on the next searcher in this chain. If there is no next, nothing is done. */ - public void fill(Result result,String summaryClass) { + public void fill(Result result, String summaryClass) { timer.sampleFill(nextIndex(), context.getDetailedDiagnostics()); - Searcher next = (Searcher)next(); // TODO: Allow but skip processors which are not searchers - if (next==null) return; + Searcher current = (Searcher)next(); // TODO: Allow but skip processors which are not searchers + if (current == null) return; try { nextProcessor(); - next.ensureFilled(result, summaryClass, this); + onInvokingFill(current, result, summaryClass); + current.ensureFilled(result, summaryClass, this); } finally { previousProcessor(); + onReturningFill(current, result, summaryClass); timer.sampleFillReturn(nextIndex(), context.getDetailedDiagnostics(), result); } } + private void onInvokingFill(Searcher searcher, Result result, String summaryClass) { + int traceFillAt = 5; + if (trace().getTraceLevel() < traceFillAt) return; + trace().trace("Invoke fill(" + summaryClass + ") on " + searcher, traceFillAt); + } + + private void onReturningFill(Searcher searcher, Result result, String summaryClass) { + int traceFillAt = 5; + if (trace().getTraceLevel() < traceFillAt) return; + trace().trace("Return fill(" + summaryClass + ") on " + searcher, traceFillAt); + } + /** Calls ping on the next search in this chain. If there is no next, a Pong is created and returned. */ public Pong ping(Ping ping) { // return this reference, not directly. It's needed for adding time data @@ -586,7 +598,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { timer.samplePing(nextIndex(), context.getDetailedDiagnostics()); Searcher next = (Searcher)next(); // TODO: Allow but skip processors which are not searchers - if (next==null) { + if (next == null) { annotationReference = new Pong(); return annotationReference; } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java index 06a4096dc68..afcf5deede7 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java @@ -14,7 +14,8 @@ import com.yahoo.search.searchchain.Execution; */ public class ExampleSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); result.hits().add(new Hit("example",1.0,"examplesearcher")); return result; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index e59808bdf5d..448098f6f06 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -94,7 +94,8 @@ public class DocumentSourceSearcher extends Searcher { addResult(q, r); } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { queryCount++; Result r; r = unFilledResults.get(getQueryKeyClone(query)); @@ -119,7 +120,8 @@ public class DocumentSourceSearcher extends Searcher { return key; } - public @Override void fill(Result result, String summaryClass, Execution execution) { + @Override + public void fill(Result result, String summaryClass, Execution execution) { Result filledResult; filledResult = completelyFilledResults.get(getQueryKeyClone(result.getQuery())); diff --git a/container-search/src/main/java/com/yahoo/search/template/.gitignore b/container-search/src/main/java/com/yahoo/search/template/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/search/template/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 176ef051968..dc01b0dd3fb 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -1267,8 +1267,10 @@ public class YqlParser implements Parser { boolean exactMatch, Language language) { String wordData = rawWord; - if (getAnnotation(ast, NFKC, Boolean.class, Boolean.TRUE, + if (getAnnotation(ast, NFKC, Boolean.class, Boolean.FALSE, "setting for whether to NFKC normalize input data")) { + // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data + // during tokenization/segmentation, as that is always turned on also on the indexing side. wordData = normalizer.normalize(wordData); } boolean fromQuery = getAnnotation(ast, IMPLICIT_TRANSFORMS, diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java b/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java index d80ff80f172..ae328886ede 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Interpretation.java @@ -360,7 +360,8 @@ public class Interpretation { return rootSpan.getTermSpans(term); } - public @Override String toString() { + @Override + public String toString() { StringBuilder sb = new StringBuilder(); Map<AnnotationClass, List<Annotations>> annotations = getAll(); Iterator<Map.Entry<AnnotationClass,List<Annotations>>> mapIterator = annotations.entrySet().iterator(); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/.gitignore b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/.gitignore +++ /dev/null diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index e27893a2b20..3a469a322ac 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -39,7 +39,7 @@ import com.yahoo.vdslib.SearchResult; * visitors in storage and collecting and merging the results. * * @author baldersheim - * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> + * @author Ulf Carlin */ @SuppressWarnings("deprecation") public class VdsStreamingSearcher extends VespaBackEndSearcher { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 9fe64e6977d..7f6ca80a65c 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -117,6 +117,16 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { setVisitorParameters(searchCluster, route); } + private static int inferSessionTraceLevel(Query query) { + int implicitLevel = 0; + if (log.isLoggable(LogLevel.SPAM)) { + implicitLevel = 9; + } else if (log.isLoggable(LogLevel.DEBUG)) { + implicitLevel = 7; + } + return Math.max(query.getTraceLevel(), implicitLevel); + } + private void setVisitorParameters(String searchCluster, Route route) { if (query.properties().getString(streamingUserid) != null) { params.setDocumentSelection("id.user==" + query.properties().getString(streamingUserid)); @@ -154,7 +164,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setMaxPending(Integer.MAX_VALUE); params.setMaxBucketsPerVisitor(Integer.MAX_VALUE); - params.setTraceLevel(query.getTraceLevel()); + params.setTraceLevel(inferSessionTraceLevel(query)); + String ordering = query.properties().getString(streamingOrdering); if (ordering != null) { @@ -297,6 +308,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } } finally { session.destroy(); + log.log(LogLevel.DEBUG, () -> session.getTrace().toString()); } query.trace(session.getTrace().toString(), false, 9); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/test/.gitignore b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/test/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/test/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java index 40d68a29a71..35beb5cb96c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java @@ -112,7 +112,8 @@ public class TilingTestCase extends junit.framework.TestCase { private static class TiledResultProducer extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=new Result(query); result.setTotalHitCount(2800000000l); @@ -256,7 +257,8 @@ public class TilingTestCase extends junit.framework.TestCase { private static class TiledResultProducer2 extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=new Result(query); result.setTotalHitCount(1); diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java index 2dd3131eb6e..58107513ee2 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -31,7 +31,8 @@ public class ClusterSearcherTestCase extends TestCase { this.hit = hit; } - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result = execution.search(query); result.hits().add(hit); return result; @@ -127,15 +128,18 @@ public class ClusterSearcherTestCase extends TestCase { super(id,searchers,hasher,false); } - public @Override Result search(Query query,Execution execution,Searcher searcher) { + @Override + public Result search(Query query,Execution execution,Searcher searcher) { return searcher.search(query,execution); } - public @Override void fill(Result result,String summaryName,Execution execution,Searcher searcher) { + @Override + public void fill(Result result,String summaryName,Execution execution,Searcher searcher) { searcher.fill(result,summaryName,execution); } - public @Override Pong ping(Ping ping,Searcher searcher) { + @Override + public Pong ping(Ping ping,Searcher searcher) { return new Execution(searcher, Execution.Context.createContextStub()).ping(ping); } diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index 06686e8777a..01fea6f0842 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -161,17 +161,20 @@ public class ClusteredConnectionTestCase extends junit.framework.TestCase { super(componentId,connections,false); } - public @Override Result search(Query query,Execution execution,Connection connection) { + @Override + public Result search(Query query,Execution execution,Connection connection) { Result result=new Result(query); result.hits().add(new Hit("from:" + connection.getResponse())); return result; } - public @Override void fill(Result result,String summary,Execution execution,Connection connection) { + @Override + public void fill(Result result,String summary,Execution execution,Connection connection) { result.hits().get(0).fields().put("filled",connection.getResponse()); } - public @Override Pong ping(Ping ping,Connection connection) { + @Override + public Pong ping(Ping ping,Connection connection) { Pong pong=new Pong(); if (connection.getResponse()==null) pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); @@ -189,7 +192,8 @@ public class ClusteredConnectionTestCase extends junit.framework.TestCase { this.hashValue = hashValue; } - public @Override int hashCode() { + @Override + public int hashCode() { return hashValue; } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index c9fbcad28f2..fe4883fd29e 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -123,6 +123,7 @@ public class GroupingParserTestCase { "sinh", "size", "sort", + "stddev", "interpolatedlookup", "sqrt", "strcat", @@ -543,6 +544,7 @@ public class GroupingParserTestCase { "Encountered \"-\" at line 1, column 29"); assertIllegalArgument("all(group(debugwait(artist, 3.3, lol)))", "Encountered \"lol\" at line 1, column 34"); + assertParse("all(group(artist) each(output(stddev(simple))))"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index 386e8346cae..8d524ae011e 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -53,16 +53,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class GroupingExecutorTestCase { - // -------------------------------------------------------------------------------- - // - // Tests - // - // -------------------------------------------------------------------------------- - @Test public void requireThatNullRequestsPass() { Result res = newExecution(new GroupingExecutor()).search(newQuery()); @@ -262,12 +256,22 @@ public class GroupingExecutorTestCase { new GroupingListHit(Arrays.asList(grp1), null))), new FillRequestThrower()); Result res = exec.search(query); + + // Fill with summary specified in grouping try { exec.fill(res); fail(); } catch (FillRequestException e) { assertEquals("bar", e.summaryClass); } + + // Fill again, with another summary + try { + exec.fill(res, "otherSummary"); + fail(); + } catch (FillRequestException e) { + assertEquals("otherSummary", e.summaryClass); + } } @Test diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java index 9fd32577737..9d527686c54 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -27,6 +27,7 @@ public class RequestBuilderTestCase { assertLayout("all(group(a) each(output(sum(b))))", "[[{ Attribute, result = [Sum] }]]"); assertLayout("all(group(a) each(each(output(summary()))))", "[[{ Attribute, result = [Hits] }]]"); assertLayout("all(group(a) each(output(xor(b))))", "[[{ Attribute, result = [Xor] }]]"); + assertLayout("all(group(a) each(output(stddev(b))))", "[[{ Attribute, result = [StandardDeviation] }]]"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java index 11698b2b70d..2dd46b54661 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java @@ -113,7 +113,8 @@ public class QueryProfileIntegrationTestCase extends junit.framework.TestCase { public static class DefaultSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); result.hits().add(new Hit("from:default")); return result; @@ -123,7 +124,8 @@ public class QueryProfileIntegrationTestCase extends junit.framework.TestCase { public static class TestSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); result.hits().add(new Hit("from:test")); return result; @@ -135,7 +137,8 @@ public class QueryProfileIntegrationTestCase extends junit.framework.TestCase { @Provides("SomeObject") public static class SettingSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { SomeObject.setTo(query,new SomeObject()); return execution.search(query); } @@ -146,7 +149,8 @@ public class QueryProfileIntegrationTestCase extends junit.framework.TestCase { @After("SomeObject") public static class ReceivingSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { assertNotNull(SomeObject.getFrom(query)); assertEquals(SomeObject.class,SomeObject.getFrom(query).getClass()); return execution.search(query); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java index 4e68ac92adc..7877e48ea50 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java @@ -194,11 +194,14 @@ public class CloningTestCase extends junit.framework.TestCase { public void set(String string) { this.string=string; } - public @Override String toString() { return string; } + @Override + public String toString() { return string; } - public @Override int hashCode() { return string.hashCode(); } + @Override + public int hashCode() { return string.hashCode(); } - public @Override boolean equals(Object other) { + @Override + public boolean equals(Object other) { if (other==this) return true; if ( ! (other instanceof MutableString)) return false; return ((MutableString)other).string.equals(string); @@ -212,7 +215,8 @@ public class CloningTestCase extends junit.framework.TestCase { super(string); } - public @Override CloneableMutableString clone() { + @Override + public CloneableMutableString clone() { try { return (CloneableMutableString)super.clone(); } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java index 4d611e3e842..17475d20292 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java @@ -92,7 +92,8 @@ public class SearchChainConfigurerTestCase { } public static abstract class TestSearcher extends Searcher { - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { return execution.search(query); } } @@ -234,7 +235,8 @@ public class SearchChainConfigurerTestCase { this.intConfig = intConfig; this.stringConfig = stringConfig; } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { return execution.search(query); } } @@ -244,7 +246,8 @@ public class SearchChainConfigurerTestCase { public IntSearcher(IntConfig intConfig) { this.intConfig = intConfig; } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { return execution.search(query); } } @@ -254,7 +257,8 @@ public class SearchChainConfigurerTestCase { public StringSearcher(StringConfig stringConfig) { this.stringConfig = stringConfig; } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { return execution.search(query); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java index 6c38df8fe08..92bc09c0162 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java @@ -13,7 +13,8 @@ public class UpdateSearcher extends com.yahoo.search.Searcher { public String test = "update"; - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); if (result==null) result=new Result(query); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java index 642d8d8cd7e..5b1d99363a3 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java @@ -219,7 +219,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { super(new ComponentId(id)); } - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); result.hits().add(new Hit(getId().stringValue() + "-" + (counter++))); return result; @@ -229,7 +230,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { public static class ForwardingSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Chain<Searcher> forwardTo=execution.context().searchChainRegistry().getChain("someChainId"); return new Execution(forwardTo,execution.context()).search(query); @@ -245,7 +247,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { super(new ComponentId(id)); } - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); Hit hit=new Hit(getId().stringValue() + "-" + counter); hit.setFillable(); @@ -253,7 +256,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { return result; } - public @Override void fill(Result result,String summaryClass,Execution execution) { + @Override + public void fill(Result result,String summaryClass,Execution execution) { result.hits().add(new Hit(getId().stringValue() + "-" + (counter++) + "-filled")); // Not something one would normally do in fill } @@ -270,7 +274,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { @After("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") public static class FillingSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result=execution.search(query); execution.fill(result); return result; @@ -282,7 +287,8 @@ public class ExecutionTestCase extends junit.framework.TestCase { @Before("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") public static class WorkflowSearcher extends Searcher { - public @Override Result search(Query query,Execution execution) { + @Override + public Result search(Query query,Execution execution) { Result result1=execution.search(query); Result result2=execution.search(query); for (Iterator<Hit> i=result2.hits().iterator(); i.hasNext();) diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java index ad0c4796549..0d87d4448f4 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.searchchain.test; import static com.yahoo.search.searchchain.test.SimpleSearchChain.searchChain; +import static org.junit.Assert.assertEquals; import java.util.ArrayList; import java.util.Arrays; @@ -18,6 +19,7 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChain; +import org.junit.Test; /** * Tests basic search chain functionality - creation, inheritance and ordering @@ -25,17 +27,15 @@ import com.yahoo.search.searchchain.SearchChain; * @author bratseth */ @SuppressWarnings("deprecation") -public class SearchChainTestCase extends junit.framework.TestCase { - - public SearchChainTestCase(String name) { - super(name); - } +public class SearchChainTestCase { + @Test public void testEmptySearchChain() { SearchChain empty = new SearchChain(new ComponentId("empty")); assertEquals("empty", empty.getId().getName()); } + @Test public void testSearchChainCreation() { assertEquals("test",searchChain.getId().stringValue()); assertEquals("test",searchChain.getId().getName()); @@ -59,22 +59,27 @@ public class SearchChainTestCase extends junit.framework.TestCase { assertEquals(new HashSet<>(correct),new HashSet<>(test)); } + @Test public void testSearchChainToStringEmpty() { assertEquals("chain 'test' []", new Chain<>(new ComponentId("test"), createSearchers(0)).toString()); } + @Test public void testSearchChainToStringVeryShort() { assertEquals("chain 'test' [s1]", new Chain<>(new ComponentId("test"),createSearchers(1)).toString()); } + @Test public void testSearchChainToStringShort() { assertEquals("chain 'test' [s1 -> s2 -> s3]", new Chain<>(new ComponentId("test"),createSearchers(3)).toString()); } + @Test public void testSearchChainToStringLong() { assertEquals("chain 'test' [s1 -> s2 -> ... -> s4]", new Chain<>(new ComponentId("test"),createSearchers(4)).toString()); } + @Test public void testSearchChainToStringVeryLong() { assertEquals("chain 'test' [s1 -> s2 -> ... -> s10]", new Chain<>(new ComponentId("test"),createSearchers(10)).toString()); } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java index a0641374a62..0833fef47bf 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java @@ -1,13 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchchain.test; +import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.trace.TraceNode; import com.yahoo.yolean.trace.TraceVisitor; +import org.junit.Test; import java.io.IOException; import java.io.StringWriter; @@ -16,38 +19,112 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + /** * Tests tracing scenarios where traces from multiple executions over the same query are involved. * * @author bratseth */ -public class TraceTestCase extends junit.framework.TestCase { +public class TraceTestCase { + @Test public void testTracingOnCorrectAPIUseNonParallel() { assertTracing(true,false); } + @Test public void testTracingOnIncorrectAPIUseNonParallel() { assertTracing(false,false); } + @Test public void testTracingOnCorrectAPIUseParallel() { assertTracing(true, true); } + @Test public void testTracingOnIncorrectAPIUseParallel() { assertTracing(false,true); } + + @Test + public void testTraceInvocationsUnfillableHits() { + final int traceLevel = 5; + Query query = new Query("?tracelevel=" + traceLevel); + Chain<Searcher> forkingChain = new Chain<>(new Tracer("tracer1"), + new Tracer("tracer2"), + new Backend("backend1", false)); + Execution execution = new Execution(forkingChain, Execution.Context.createContextStub()); + Result result = execution.search(query); + execution.fill(result, "mySummary"); + + Iterator<String> trace = collectTrace(query).iterator(); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + trace.next(); // (properties trace: not checked) + assertEquals(" (level start)", trace.next()); + assertEquals(" Invoke searcher 'tracer1'", trace.next()); + assertEquals(" During tracer1: 0", trace.next()); + assertEquals(" Invoke searcher 'tracer2'", trace.next()); + assertEquals(" During tracer2: 0", trace.next()); + assertEquals(" Invoke searcher 'backend1'", trace.next()); + assertEquals(" Return searcher 'backend1'", trace.next()); + assertEquals(" Return searcher 'tracer2'", trace.next()); + assertEquals(" Return searcher 'tracer1'", trace.next()); + assertEquals(" Invoke fill(mySummary) on searcher 'tracer1'", trace.next()); + assertEquals(" Ignoring fill(mySummary): Hits are unfillable: result.hits().getFilled()=null", trace.next()); + assertEquals(" Return fill(mySummary) on searcher 'tracer1'", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals("(level end)", trace.next()); + assertFalse(trace.hasNext()); + } - @SuppressWarnings("deprecation") - public void assertTracing(boolean carryOverContext,boolean parallel) { - Query query=new Query("?tracelevel=1"); + @Test + public void testTraceInvocationsFillableHits() { + final int traceLevel = 5; + Query query = new Query("?tracelevel=" + traceLevel); + Chain<Searcher> forkingChain = new Chain<>(new Tracer("tracer1"), + new Tracer("tracer2"), + new Backend("backend1", true)); + Execution execution = new Execution(forkingChain, Execution.Context.createContextStub()); + Result result = execution.search(query); + execution.fill(result, "mySummary"); + + Iterator<String> trace = collectTrace(query).iterator(); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + trace.next(); // (properties trace: not checked) + assertEquals(" (level start)", trace.next()); + assertEquals(" Invoke searcher 'tracer1'", trace.next()); + assertEquals(" During tracer1: 0", trace.next()); + assertEquals(" Invoke searcher 'tracer2'", trace.next()); + assertEquals(" During tracer2: 0", trace.next()); + assertEquals(" Invoke searcher 'backend1'", trace.next()); + assertEquals(" Return searcher 'backend1'", trace.next()); + assertEquals(" Return searcher 'tracer2'", trace.next()); + assertEquals(" Return searcher 'tracer1'", trace.next()); + assertEquals(" Invoke fill(mySummary) on searcher 'tracer1'", trace.next()); + assertEquals(" Invoke fill(mySummary) on searcher 'tracer2'", trace.next()); + assertEquals(" Invoke fill(mySummary) on searcher 'backend1'", trace.next()); + assertEquals(" Return fill(mySummary) on searcher 'backend1'", trace.next()); + assertEquals(" Return fill(mySummary) on searcher 'tracer2'", trace.next()); + assertEquals(" Return fill(mySummary) on searcher 'tracer1'", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals("(level end)", trace.next()); + assertFalse(trace.hasNext()); + } + + private void assertTracing(boolean carryOverContext, boolean parallel) { + Query query = new Query("?tracelevel=1"); query.trace("Before execution",1); - Chain<Searcher> forkingChain=new Chain<>(new Tracer("forker"),new Forker(carryOverContext,parallel,new Tracer("branch 1"),new Tracer("branch 2"))); + Chain<Searcher> forkingChain = new Chain<>(new Tracer("forker"), + new Forker(carryOverContext, parallel, + new Tracer("branch 1") , + new Tracer("branch 2"))); new Execution(forkingChain, Execution.Context.createContextStub()).search(query); - // printTrace(query); - if (carryOverContext) assertTraceWithChildExecutionMessages(query); else if (parallel) @@ -60,20 +137,20 @@ public class TraceTestCase extends junit.framework.TestCase { // The valid and usual trace private void assertTraceWithChildExecutionMessages(Query query) { - Iterator<String> trace=collectTrace(query).iterator(); - assertEquals("(level start)",trace.next()); - assertEquals(" No query profile is used",trace.next()); - assertEquals(" Before execution",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During forker: 0",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During branch 1: 0",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level start)",trace.next()); + Iterator<String> trace = collectTrace(query).iterator(); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + assertEquals(" Before execution", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During forker: 0", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During branch 1: 0", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level start)", trace.next()); assertEquals(" During branch 2: 0", trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals("(level end)",trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals("(level end)", trace.next()); assertFalse(trace.hasNext()); } @@ -81,19 +158,19 @@ public class TraceTestCase extends junit.framework.TestCase { // where the message of the execution subtrees is empty rather than "child execution". This is fine. private void assertTrace(Query query) { Iterator<String> trace=collectTrace(query).iterator(); - assertEquals("(level start)",trace.next()); - assertEquals(" No query profile is used",trace.next()); - assertEquals(" Before execution",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During forker: 0",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During branch 1: 0",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level start)",trace.next()); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + assertEquals(" Before execution", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During forker: 0", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During branch 1: 0", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level start)", trace.next()); assertEquals(" During branch 2: 0", trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals("(level end)",trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals("(level end)", trace.next()); assertFalse(trace.hasNext()); } @@ -101,27 +178,27 @@ public class TraceTestCase extends junit.framework.TestCase { // but all the trace messages are present. private void assertIncorrectlyNestedTrace(Query query) { Iterator<String> trace=collectTrace(query).iterator(); - assertEquals("(level start)",trace.next()); - assertEquals(" No query profile is used",trace.next()); - assertEquals(" Before execution",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During forker: 0",trace.next()); - assertEquals(" (level start)",trace.next()); - assertEquals(" During branch 1: 0",trace.next()); - assertEquals(" (level start)",trace.next()); + assertEquals("(level start)", trace.next()); + assertEquals(" No query profile is used", trace.next()); + assertEquals(" Before execution", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During forker: 0", trace.next()); + assertEquals(" (level start)", trace.next()); + assertEquals(" During branch 1: 0", trace.next()); + assertEquals(" (level start)", trace.next()); assertEquals(" During branch 2: 0", trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals(" (level end)",trace.next()); - assertEquals("(level end)",trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals(" (level end)", trace.next()); + assertEquals("(level end)", trace.next()); assertFalse(trace.hasNext()); } private void assertCorrectRendering(Query query) { try { - StringWriter writer=new StringWriter(); + StringWriter writer = new StringWriter(); query.getContext(false).render(writer); - String expected= + String expected = "<meta type=\"context\">\n" + "\n" + " <p>No query profile is used</p>\n" + @@ -130,7 +207,7 @@ public class TraceTestCase extends junit.framework.TestCase { "\n" + " <p>\n" + " <p>During forker: 0"; - assertEquals(expected,writer.toString().substring(0,expected.length())); + assertEquals(expected, writer.toString().substring(0, expected.length())); } catch (IOException e) { throw new RuntimeException(e); @@ -138,15 +215,15 @@ public class TraceTestCase extends junit.framework.TestCase { } private List<String> collectTrace(Query query) { - TraceCollector collector=new TraceCollector(); + TraceCollector collector = new TraceCollector(); query.getContext(false).getTrace().accept(collector); return collector.trace(); } private static class TraceCollector extends TraceVisitor { - private List<String> trace=new ArrayList<>(); - private StringBuilder indent=new StringBuilder(); + private List<String> trace = new ArrayList<>(); + private StringBuilder indent = new StringBuilder(); @Override public void entering(TraceNode node) { @@ -156,14 +233,14 @@ public class TraceTestCase extends junit.framework.TestCase { @Override public void leaving(TraceNode end) { - indent.setLength(indent.length()-2); + indent.setLength(indent.length() - 2); trace.add(indent + "(level end)"); } @Override public void visit(TraceNode node) { if (node.isRoot()) return; - if (node.payload()==null) return; + if (node.payload() == null) return; trace.add(indent + node.payload().toString()); } @@ -173,15 +250,16 @@ public class TraceTestCase extends junit.framework.TestCase { private static class Tracer extends Searcher { private String name; - private int counter=0; + private int counter = 0; public Tracer(String name) { - this.name=name; + super(new ComponentId(name)); + this.name = name; } @Override public Result search(Query query, Execution execution) { - query.trace("During " + name + ": " + (counter++) ,1); + query.trace("During " + name + ": " + (counter++),1); return execution.search(query); } } @@ -196,20 +274,21 @@ public class TraceTestCase extends junit.framework.TestCase { /** If true, simulate parallel execution by cloning the query */ private boolean parallel; - public Forker(boolean carryOverContext,boolean parallel,Searcher ... branches) { - this.carryOverContext=carryOverContext; - this.parallel=parallel; - this.branches=Arrays.asList(branches); + public Forker(boolean carryOverContext, boolean parallel, Searcher ... branches) { + this.carryOverContext = carryOverContext; + this.parallel = parallel; + this.branches = Arrays.asList(branches); } @SuppressWarnings("deprecation") @Override public Result search(Query query, Execution execution) { - Result result=execution.search(query); + Result result = execution.search(query); for (Searcher branch : branches) { - Query branchQuery=parallel ? query.clone() : query; - Result branchResult= - ( carryOverContext ? new Execution(branch,execution.context()) : new Execution(branch, Execution.Context.createContextStub())).search(branchQuery); + Query branchQuery = parallel ? query.clone() : query; + Result branchResult = + ( carryOverContext ? new Execution(branch, execution.context()) : + new Execution(branch, Execution.Context.createContextStub())).search(branchQuery); result.hits().add(branchResult.hits()); result.mergeWith(branchResult); } @@ -218,4 +297,28 @@ public class TraceTestCase extends junit.framework.TestCase { } + private static class Backend extends Searcher { + + private final boolean fillableHits; + + public Backend(String name, boolean fillableHits) { + super(new ComponentId(name)); + this.fillableHits = fillableHits; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + Hit hit0 = new Hit("hit:0"); + Hit hit1 = new Hit("hit:1"); + if (fillableHits) { + hit0.setFillable(); + hit1.setFillable(); + } + result.hits().add(hit0); + result.hits().add(hit1); + return result; + } + } + } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 32facd7a754..42f285cf096 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -354,9 +354,10 @@ public class YqlParserTestCase { assertEquals("\u00e5", getRootWord("select foo from bar where baz contains " + "([ {\"nfkc\": true} ]\"a\\u030a\");").getWord()); - assertEquals("\u00e5", + assertEquals("No NKFC by default", + "a\u030a", getRootWord("select foo from bar where baz contains " + - "\"a\\u030a\";").getWord()); + "(\"a\\u030a\");").getWord()); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java index 1fe62c2cd35..0932b202dac 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java @@ -114,7 +114,8 @@ public class MetricsSearcherTestCase { this.implicitlyCreateContext = implicitlyCreateContext; } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { if (implicitlyCreateContext) { String loadType = query.properties().getString("streaming.loadtype"); assignContextProperties(query, loadType); diff --git a/dist/post_install.sh b/dist/post_install.sh index ddaa5f0ff00..a5cb48c2d9e 100755 --- a/dist/post_install.sh +++ b/dist/post_install.sh @@ -67,4 +67,39 @@ ln -s $PREFIX/lib/jars/orchestrator-jar-with-dependencies.jar $INSTALLPATH/conf/ ln -s $PREFIX/lib/jars/node-repository-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/node-repository.jar ln -s $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/zkfacade.jar ln -s $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models -ln -s storaged-bin $INSTALLPATH/sbin/distributord-bin +ln -s vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin + +# Temporary when renaming binaries in fnet +ln -s vespa-rpc-info $INSTALLPATH/bin/rpc_info +ln -s vespa-rpc-invoke $INSTALLPATH/bin/rpc_invoke +ln -s vespa-rpc-proxy $INSTALLPATH/bin/rpc_proxy + +# Temporary when renaming binaries in fsa +ln -s vespa-fsadump $INSTALLPATH/bin/fsadump +ln -s vespa-fsainfo $INSTALLPATH/bin/fsainfo +ln -s vespa-makefsa $INSTALLPATH/bin/makefsa + +# Temporary when renaming binaries in searchcore +ln -s vespa-proton-bin $INSTALLPATH/sbin/proton-bin +ln -s vespa-fdispatch-bin $INSTALLPATH/sbin/fdispatch-bin +ln -s vespa-verify-ranksetup-bin $INSTALLPATH/bin/verify_ranksetup-bin +ln -s vespa-proton $INSTALLPATH/sbin/proton +ln -s vespa-fdispatch $INSTALLPATH/sbin/fdispatch +ln -s vespa-verify-ranksetup $INSTALLPATH/bin/verify_ranksetup + +# Temporary when renaming binaries in storageserver +ln -s vespa-storaged-bin $INSTALLPATH/sbin/storaged-bin +ln -s vespa-distributord-bin $INSTALLPATH/sbin/distributord-bin +ln -s vespa-storaged $INSTALLPATH/sbin/storaged +ln -s vespa-distributord $INSTALLPATH/sbin/distributord + +# Temporary when renaming binaries in vespalog +ln -s vespa-log-conv $INSTALLPATH/bin/vlogconv +ln -s vespa-logctl $INSTALLPATH/bin/logctl +ln -s vespa-logfmt $INSTALLPATH/bin/logfmt +ln -s vespa-log-replay $INSTALLPATH/bin/vlogreplay + +# Temporary when renaming binaries in memfilepersistence +ln -s vespa-dump-slotfile $INSTALLPATH/bin/dumpslotfile +ln -s vespa-vds-disktool-bin $INSTALLPATH/bin/vdsdisktool-bin +ln -s vespa-vds-disktool $INSTALLPATH/bin/vdsdisktool diff --git a/docker-api/OWNERS b/docker-api/OWNERS index 7acec855f4a..e030acdbc5b 100644 --- a/docker-api/OWNERS +++ b/docker-api/OWNERS @@ -1,2 +1 @@ freva -dybis diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java index 94940601f80..01a28518e97 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; public class MetricReceiverWrapper { // Application names used public static final String APPLICATION_DOCKER = "docker"; + public static final String APPLICATION_NODE = "vespa.node"; public static final String APPLICATION_HOST_LIFE = "host_life"; private final static ObjectMapper objectMapper = new ObjectMapper(); diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build index 75dee2bd0a7..a0b935093eb 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile.build @@ -25,6 +25,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp perl \ perl-Env \ perl-JSON \ + perl-IO-Socket-IP \ sudo \ vespa-boost-devel \ vespa-libtorrent-devel \ diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run index 0f57da4b111..5f0dd1fd0cc 100644 --- a/docker/Dockerfile.run +++ b/docker/Dockerfile.run @@ -15,6 +15,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp perl \ perl-Env \ perl-JSON \ + perl-IO-Socket-IP \ libatomic \ net-tools \ less \ diff --git a/docker/README.md b/docker/README.md index 04c45470017..31766259365 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,17 +6,12 @@ *On Linux, the default storage device is devicemapper with loopback device and max 10GB container size. This size is too small for a full build. Please see [here](http://www.projectatomic.io/blog/2016/03/daemon_option_basedevicesize/) and [here](http://www.projectatomic.io/blog/2015/06/notes-on-fedora-centos-and-docker-storage-drivers/) to overcome this limitation.* + ## Building the Vespa RPM Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this source code. The produced rpms will be available in this folder after compiliation. -### Troubleshooting -If the build fails, start from scratch: ```docker rmi -f vesparun vespabuild``` - then build again. Clean local docker if docker image disk full: -- ```docker rm -v $(docker ps -a -q -f status=exited)``` -- ```docker rmi $(docker images -f "dangling=true" -q)``` - -_Directory renamed before its status could be extracted_ can be caused by [1219](https://github.com/docker/for-mac/issues/1219) ## Running Vespa Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa. @@ -25,8 +20,18 @@ This will create a Docker image which has the rpms from the build step (or downl *On OS X, the container runs inside the Docker VM. Execute ```docker-machine ssh vespa-docker-machine``` to enter the VM. The services can also be reached directly from the host on the IP given by ```docker-machine ip vespa-docker-machine```* + ## Building Vespa inside a Docker container Execute ```./enter-build-container.sh``` to enter the Vespa build environment inside a Docker container. The container is entered at the root of the Vespa source repository. Follow the build sections in [README.md](https://github.com/yahoo/vespa/blob/master/README.md) to build and test. + +## Troubleshooting +- Use ```docker logs CONTAINER``` for output - useful if the commands above fail + +- If the build fails, start from scratch: ```docker rmi -f vesparun vespabuild``` - then build again. Clean local docker if docker image disk full: + - ```docker rm -v $(docker ps -a -q -f status=exited)``` + - ```docker rmi $(docker images -f "dangling=true" -q)``` + +- _Directory renamed before its status could be extracted_ can be caused by [1219](https://github.com/docker/for-mac/issues/1219) - workaround (from the issue): "It may be an overlay storage driver issue - you can add ```{"storage-driver":"aufs"}``` in the advanced daemon preferences pane and see if that makes a difference." diff --git a/docproc/src/main/java/com/yahoo/docproc/configuration/.gitignore b/docproc/src/main/java/com/yahoo/docproc/configuration/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/docproc/src/main/java/com/yahoo/docproc/configuration/.gitignore +++ /dev/null diff --git a/docproc/src/main/java/com/yahoo/docproc/documentstatus/.gitignore b/docproc/src/main/java/com/yahoo/docproc/documentstatus/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/docproc/src/main/java/com/yahoo/docproc/documentstatus/.gitignore +++ /dev/null diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java index 816fcd7bdf7..a989422ab92 100644 --- a/document/src/main/java/com/yahoo/document/PositionDataType.java +++ b/document/src/main/java/com/yahoo/document/PositionDataType.java @@ -73,10 +73,18 @@ public final class PositionDataType { DegreesParser d = new DegreesParser(str); return valueOf((int)(d.longitude * 1000000), (int)(d.latitude * 1000000)); } catch (IllegalArgumentException e) { - // empty + try { + String[] arr = str.split(";"); + if (arr.length == 2) { + int x = Integer.parseInt(arr[0]); + int y = Integer.parseInt(arr[1]); + return valueOf(x, y); + } + } catch (NumberFormatException nfe) { + // empty + } + throw new IllegalArgumentException("Could not parse '"+str+"' as geo coordinates: "+e.getMessage()); } - String[] arr = str.split(";", 2); - return valueOf(Integer.parseInt(arr[0]), Integer.parseInt(arr[1])); } public static IntegerFieldValue getXValue(FieldValue pos) { diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index df09b312738..4d18c5ca51a 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -95,5 +95,10 @@ public class TensorFieldValue extends FieldValue { return true; } + @Override + public Object getWrappedValue() { + return tensor.orElse(null); + } + } diff --git a/document/src/main/java/com/yahoo/document/declaration/.gitignore b/document/src/main/java/com/yahoo/document/declaration/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/document/src/main/java/com/yahoo/document/declaration/.gitignore +++ /dev/null diff --git a/document/src/main/java/com/yahoo/documentmodel/.gitignore b/document/src/main/java/com/yahoo/documentmodel/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/document/src/main/java/com/yahoo/documentmodel/.gitignore +++ /dev/null diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp index c7e34430393..7a6cf3fdf46 100644 --- a/document/src/tests/annotation/annotation_test.cpp +++ b/document/src/tests/annotation/annotation_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for annotation. -#include <vespa/fastos/fastos.h> #include <stdlib.h> #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/annotation/annotation.h> diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index 7f31e0c3b3a..0535edc495b 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -4,6 +4,7 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> using vespalib::nbostream; diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp index adda311e01a..7f17685f84e 100644 --- a/document/src/tests/base/documentid_test.cpp +++ b/document/src/tests/base/documentid_test.cpp @@ -2,6 +2,7 @@ // Unit tests for documentid. #include <vespa/document/base/documentid.h> +#include <vespa/document/base/idstringexception.h> #include <vespa/vespalib/testkit/testapp.h> using namespace document; diff --git a/document/src/tests/bucketselectortest.cpp b/document/src/tests/bucketselectortest.cpp index f04816cba0d..e91243cbcb8 100644 --- a/document/src/tests/bucketselectortest.cpp +++ b/document/src/tests/bucketselectortest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/bucket/bucketselector.h> #include <vespa/document/bucket/bucketid.h> diff --git a/document/src/tests/buckettest.cpp b/document/src/tests/buckettest.cpp index 505ba5196c6..be5f9cb6cdb 100644 --- a/document/src/tests/buckettest.cpp +++ b/document/src/tests/buckettest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cppunit/extensions/HelperMacros.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> diff --git a/document/src/tests/documentcalculatortestcase.cpp b/document/src/tests/documentcalculatortestcase.cpp index a292302ad61..bc112c80a8c 100644 --- a/document/src/tests/documentcalculatortestcase.cpp +++ b/document/src/tests/documentcalculatortestcase.cpp @@ -9,6 +9,7 @@ #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/fieldvalue/floatfieldvalue.h> +#include <vespa/document/select/variablemap.h> #include <vespa/vespalib/util/exceptions.h> namespace document { @@ -52,46 +53,42 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentCalculatorTest); void DocumentCalculatorTest::testConstant() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "4.0"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables))); } void DocumentCalculatorTest::testSimple() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "(3 + 5) / 2"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables))); } void DocumentCalculatorTest::testVariables() { - DocumentCalculator::VariableMap variables; - variables["x"] = 3.0; - variables["y"] = 5.0; + auto variables = std::make_unique<select::VariableMap>(); + (*variables)["x"] = 3.0; + (*variables)["y"] = 5.0; DocumentCalculator calc(getRepo(), "($x + $y) / 2"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables))); } void DocumentCalculatorTest::testFields() { - DocumentCalculator::VariableMap variables; - variables["x"] = 3.0; - variables["y"] = 5.0; + auto variables = std::make_unique<select::VariableMap>(); + (*variables)["x"] = 3.0; + (*variables)["y"] = 5.0; DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1" ".hfloatval) / testdoctype1.headerlongval"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); doc.setValue(doc.getField("headerval"), IntFieldValue(5)); doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0)); doc.setValue(doc.getField("headerlongval"), LongFieldValue(2)); @@ -100,14 +97,13 @@ DocumentCalculatorTest::testFields() { void DocumentCalculatorTest::testFieldsDivZero() { - DocumentCalculator::VariableMap variables; - variables["x"] = 3.0; - variables["y"] = 5.0; + auto variables = std::make_unique<select::VariableMap>(); + (*variables)["x"] = 3.0; + (*variables)["y"] = 5.0; DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1" ".hfloatval) / testdoctype1.headerlongval"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); doc.setValue(doc.getField("headerval"), IntFieldValue(5)); doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0)); doc.setValue(doc.getField("headerlongval"), LongFieldValue(0)); @@ -121,11 +117,10 @@ DocumentCalculatorTest::testFieldsDivZero() { void DocumentCalculatorTest::testDivideByZero() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "(3 + 5) / 0"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); try { calc.evaluate(doc, std::move(variables)); CPPUNIT_ASSERT(false); @@ -136,11 +131,10 @@ DocumentCalculatorTest::testDivideByZero() { void DocumentCalculatorTest::testModByZero() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "(3 + 5) % 0"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); try { calc.evaluate(doc, std::move(variables)); CPPUNIT_ASSERT(false); @@ -151,12 +145,11 @@ DocumentCalculatorTest::testModByZero() { void DocumentCalculatorTest::testFieldNotSet() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1" ".hfloatval) / testdoctype1.headerlongval"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0)); doc.setValue(doc.getField("headerlongval"), LongFieldValue(2)); try { @@ -169,13 +162,12 @@ DocumentCalculatorTest::testFieldNotSet() { void DocumentCalculatorTest::testFieldNotFound() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "(testdoctype1.mynotfoundfield + testdoctype1" ".hfloatval) / testdoctype1.headerlongval"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0)); doc.setValue(doc.getField("headerlongval"), LongFieldValue(2)); try { @@ -188,11 +180,10 @@ DocumentCalculatorTest::testFieldNotFound() { void DocumentCalculatorTest::testByteSubtractionZeroResult() { - DocumentCalculator::VariableMap variables; + auto variables = std::make_unique<select::VariableMap>(); DocumentCalculator calc(getRepo(), "testdoctype1.byteval - 3"); - Document doc(*_testRepo.getDocumentType("testdoctype1"), - DocumentId("doc:test:foo")); + Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo")); doc.setValue(doc.getField("byteval"), ByteFieldValue(3)); CPPUNIT_ASSERT_EQUAL(0.0, calc.evaluate(doc, std::move(variables))); } diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 6957634cbde..7fb7703c04a 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/visitor.h> diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index b100487a94e..1aa6f5fcaa2 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -5,12 +5,14 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/document/datatype/annotationreferencedatatype.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/util/serializableexceptions.h> +#include <vespa/document/util/bytebuffer.h> using vespalib::nbostream; @@ -18,6 +20,8 @@ using namespace document::config_builder; namespace document { +using namespace fieldvalue; + struct DocumentTest : public CppUnit::TestFixture { void testTraversing(); void testFieldPath(); @@ -75,7 +79,7 @@ void DocumentTest::testSizeOf() CPPUNIT_ASSERT_EQUAL(120ul, sizeof(Document)); CPPUNIT_ASSERT_EQUAL(64ul, sizeof(StructFieldValue)); CPPUNIT_ASSERT_EQUAL(16ul, sizeof(StructuredFieldValue)); - CPPUNIT_ASSERT_EQUAL(120ul, sizeof(SerializableArray)); + CPPUNIT_ASSERT_EQUAL(64ul, sizeof(SerializableArray)); } void DocumentTest::testFieldPath() @@ -104,7 +108,7 @@ void DocumentTest::testFieldPath() } } -class Handler : public FieldValue::IteratorHandler { +class Handler : public fieldvalue::IteratorHandler { public: Handler(); ~Handler(); @@ -184,7 +188,7 @@ void DocumentTest::testTraversing() std::string("<P<P<PP[PPP][<PP><PP>]>>>")); } -class VariableIteratorHandler : public FieldValue::IteratorHandler { +class VariableIteratorHandler : public IteratorHandler { public: VariableIteratorHandler(); ~VariableIteratorHandler(); @@ -267,16 +271,16 @@ DocumentTest::testVariables() } -class ModifyIteratorHandler : public FieldValue::IteratorHandler { +class ModifyIteratorHandler : public IteratorHandler { public: ModificationStatus doModify(FieldValue& fv) override { StringFieldValue* sfv = dynamic_cast<StringFieldValue*>(&fv); if (sfv != NULL) { *sfv = std::string("newvalue"); - return MODIFIED; + return ModificationStatus::MODIFIED; } - return NOT_MODIFIED; + return ModificationStatus::NOT_MODIFIED; }; bool onComplex(const Content&) override { diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index baa777ceca1..e3cab489aab 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -24,6 +24,7 @@ #include <vespa/eval/tensor/tensor_factory.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/exception.h> using namespace document::config_builder; using vespalib::tensor::Tensor; diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 4a23f828921..2cf29e4cc53 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/node.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vdstestlib/cppunit/macros.h> @@ -21,6 +22,8 @@ using namespace document::config_builder; namespace document { +using namespace fieldvalue; + struct FieldPathUpdateTestCase : public CppUnit::TestFixture { DocumentTypeRepo::SP _repo; DocumentType _foobar_type; @@ -259,7 +262,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { struct TestFieldPathUpdate : FieldPathUpdate { - struct TestIteratorHandler : FieldValue::IteratorHandler + struct TestIteratorHandler : fieldvalue::IteratorHandler { TestIteratorHandler(std::string& str) : _str(str) {} @@ -272,7 +275,7 @@ struct TestFieldPathUpdate : FieldPathUpdate _str += ';'; } _str += ss.str(); - return NOT_MODIFIED; + return ModificationStatus::NOT_MODIFIED; } bool onComplex(const Content&) override { return false; } @@ -288,9 +291,9 @@ struct TestFieldPathUpdate : FieldPathUpdate TestFieldPathUpdate(const TestFieldPathUpdate& other); - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override + std::unique_ptr<IteratorHandler> getIteratorHandler(Document&) const override { - return std::unique_ptr<FieldValue::IteratorHandler>( + return std::unique_ptr<IteratorHandler>( new TestIteratorHandler(_str)); } diff --git a/document/src/tests/fieldsettest.cpp b/document/src/tests/fieldsettest.cpp index 41a530c4b17..647535410bd 100644 --- a/document/src/tests/fieldsettest.cpp +++ b/document/src/tests/fieldsettest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/document/fieldvalue/fieldvalues.h> diff --git a/document/src/tests/fieldvalue/fieldvalue_test.cpp b/document/src/tests/fieldvalue/fieldvalue_test.cpp index 7fecb42417e..18a16f3c101 100644 --- a/document/src/tests/fieldvalue/fieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/fieldvalue_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("fieldvalue_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/longfieldvalue.h> diff --git a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp index 94b0f406696..a444c17a037 100644 --- a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp +++ b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicatefieldvalue_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> diff --git a/document/src/tests/forcelinktest.cpp b/document/src/tests/forcelinktest.cpp index c62f327585f..121e900b6e7 100644 --- a/document/src/tests/forcelinktest.cpp +++ b/document/src/tests/forcelinktest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/forcelink.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/document/src/tests/globalidtest.cpp b/document/src/tests/globalidtest.cpp index d0d066e5510..cc4e0a6b687 100644 --- a/document/src/tests/globalidtest.cpp +++ b/document/src/tests/globalidtest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cppunit/extensions/HelperMacros.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> diff --git a/document/src/tests/heapdebuggerlinux.cpp b/document/src/tests/heapdebuggerlinux.cpp index 5afe583c5e1..c9e266602a1 100644 --- a/document/src/tests/heapdebuggerlinux.cpp +++ b/document/src/tests/heapdebuggerlinux.cpp @@ -13,7 +13,6 @@ // These are necessary to make it compile.... #define _MALLOC_INTERNAL #define MALLOC_HOOKS -#include <vespa/fastos/fastos.h> #include <stdio.h> #define _LIBC #include <malloc.h> diff --git a/document/src/tests/heapdebuggerother.cpp b/document/src/tests/heapdebuggerother.cpp index 641951f1883..f5522488348 100644 --- a/document/src/tests/heapdebuggerother.cpp +++ b/document/src/tests/heapdebuggerother.cpp @@ -9,7 +9,6 @@ * ALL RIGHTS RESERVED */ -#include <vespa/fastos/fastos.h> #include <stdlib.h> #include "heapdebugger.h" diff --git a/document/src/tests/orderingselectortest.cpp b/document/src/tests/orderingselectortest.cpp index b9cf7dc358a..3157dfbf281 100644 --- a/document/src/tests/orderingselectortest.cpp +++ b/document/src/tests/orderingselectortest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/select/orderingselector.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/testdocrepo.h> diff --git a/document/src/tests/positiontypetest.cpp b/document/src/tests/positiontypetest.cpp index 33398e9fa00..14e070b371e 100644 --- a/document/src/tests/positiontypetest.cpp +++ b/document/src/tests/positiontypetest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/positiondatatype.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/document/src/tests/predicate/predicate_builder_test.cpp b/document/src/tests/predicate/predicate_builder_test.cpp index 1e32aa2cc51..da3541e9536 100644 --- a/document/src/tests/predicate/predicate_builder_test.cpp +++ b/document/src/tests/predicate/predicate_builder_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_builder_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_builder.h> diff --git a/document/src/tests/predicate/predicate_printer_test.cpp b/document/src/tests/predicate/predicate_printer_test.cpp index 8525fb8e2db..ff9a7d3fee9 100644 --- a/document/src/tests/predicate/predicate_printer_test.cpp +++ b/document/src/tests/predicate/predicate_printer_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_printer_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_printer.h> diff --git a/document/src/tests/predicate/predicate_test.cpp b/document/src/tests/predicate/predicate_test.cpp index 3223b628c6a..2ac7c921a10 100644 --- a/document/src/tests/predicate/predicate_test.cpp +++ b/document/src/tests/predicate/predicate_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 15fd6e00959..018408a2d02 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -4,6 +4,7 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> using vespalib::nbostream; diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp index 01e19ae946b..b41dfad7b36 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/document/src/tests/serialization/compression_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for annotation serialization. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> diff --git a/document/src/tests/stringtokenizertest.cpp b/document/src/tests/stringtokenizertest.cpp index 5ac41992621..0432db58624 100644 --- a/document/src/tests/stringtokenizertest.cpp +++ b/document/src/tests/stringtokenizertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cppunit/extensions/HelperMacros.h> #include <iostream> #include <set> diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp index d6e585e0987..1b2fa17a79a 100644 --- a/document/src/tests/structfieldvaluetest.cpp +++ b/document/src/tests/structfieldvaluetest.cpp @@ -1,10 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> using vespalib::nbostream; using document::config_builder::Struct; diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp index b61cb4895df..5adab10e2cc 100644 --- a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp +++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("fieldvalue_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/eval/tensor/tensor.h> diff --git a/document/src/tests/testdocmantest.cpp b/document/src/tests/testdocmantest.cpp index 14140b4b508..2decb984556 100644 --- a/document/src/tests/testdocmantest.cpp +++ b/document/src/tests/testdocmantest.cpp @@ -2,7 +2,6 @@ /* $Id$*/ -#include <vespa/fastos/fastos.h> #include <iostream> #include <set> #include <sstream> diff --git a/document/src/tests/teststringutil.cpp b/document/src/tests/teststringutil.cpp index 9d04372a19b..54c7457a10d 100644 --- a/document/src/tests/teststringutil.cpp +++ b/document/src/tests/teststringutil.cpp @@ -5,7 +5,6 @@ * @version $Id$ */ -#include <vespa/fastos/fastos.h> #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/document/src/tests/testxml.cpp b/document/src/tests/testxml.cpp index da328c7a63f..a9abae531fa 100644 --- a/document/src/tests/testxml.cpp +++ b/document/src/tests/testxml.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* $Id$*/ #include <vespa/document/base/testdocrepo.h> #include <cppunit/extensions/HelperMacros.h> @@ -8,6 +7,7 @@ #include <vespa/document/update/addvalueupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/removevalueupdate.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/testkit/test_kit.h> diff --git a/document/src/tests/urltypetest.cpp b/document/src/tests/urltypetest.cpp index 0aa447b9e40..6abe5e66a0c 100644 --- a/document/src/tests/urltypetest.cpp +++ b/document/src/tests/urltypetest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/urldatatype.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 81b98da297f..68b94cd2f16 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -4,6 +4,7 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> using vespalib::nbostream; diff --git a/document/src/vespa/document/annotation/spantree.cpp b/document/src/vespa/document/annotation/spantree.cpp index 199040ea8f6..61c7ac48817 100644 --- a/document/src/vespa/document/annotation/spantree.cpp +++ b/document/src/vespa/document/annotation/spantree.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "spantree.h" -#include "annotation.h" #include "spannode.h" -#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/asciistream.h> using std::unique_ptr; diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index d53e9002598..06558cf6887 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -4,6 +4,7 @@ #include <vespa/document/annotation/annotation.h> #include <vector> +#include <cassert> namespace document { class SpanNode; diff --git a/document/src/vespa/document/base/documentcalculator.cpp b/document/src/vespa/document/base/documentcalculator.cpp index 7a83d80764c..24afdee3bd1 100644 --- a/document/src/vespa/document/base/documentcalculator.cpp +++ b/document/src/vespa/document/base/documentcalculator.cpp @@ -5,6 +5,7 @@ #include <vespa/document/select/compare.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/valuenode.h> +#include <vespa/document/select/variablemap.h> #include <vespa/vespalib/util/exceptions.h> namespace document { @@ -21,13 +22,13 @@ DocumentCalculator::DocumentCalculator( DocumentCalculator::~DocumentCalculator() { } double -DocumentCalculator::evaluate(const Document& doc, VariableMap && variables) +DocumentCalculator::evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables) { select::Compare& compare(static_cast<select::Compare&>(*_selectionNode)); const select::ValueNode& left = compare.getLeft(); select::Context context(doc); - context._variables = std::move(variables); + context.setVariableMap(std::move(variables)); std::unique_ptr<select::Value> value = left.getValue(context); select::NumberValue* num = dynamic_cast<select::NumberValue*>(value.get()); diff --git a/document/src/vespa/document/base/documentcalculator.h b/document/src/vespa/document/base/documentcalculator.h index 7e1feaf5688..fc9eb73dea2 100644 --- a/document/src/vespa/document/base/documentcalculator.h +++ b/document/src/vespa/document/base/documentcalculator.h @@ -2,18 +2,18 @@ #pragma once #include <vespa/document/select/node.h> -#include <vespa/vespalib/stllike/hash_map.h> namespace document { + +namespace select { class VariableMap; } + class DocumentTypeRepo; class DocumentCalculator { public: - using VariableMap = vespalib::hash_map<vespalib::string, double>; - DocumentCalculator(const DocumentTypeRepo& repo, const vespalib::string& expression); ~DocumentCalculator(); - double evaluate(const Document& doc, VariableMap && variables); + double evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables); private: std::unique_ptr<select::Node> _selectionNode; diff --git a/document/src/vespa/document/base/exceptions.cpp b/document/src/vespa/document/base/exceptions.cpp index 3e5c464b8b4..dbd3f2bd998 100644 --- a/document/src/vespa/document/base/exceptions.cpp +++ b/document/src/vespa/document/base/exceptions.cpp @@ -3,6 +3,7 @@ #include "exceptions.h" #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/util/stringfmt.h> namespace document { diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp index d62623b64b9..d00fd42081c 100644 --- a/document/src/vespa/document/base/field.cpp +++ b/document/src/vespa/document/base/field.cpp @@ -4,6 +4,7 @@ #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/bobhash.h> namespace document { diff --git a/document/src/vespa/document/base/forcelink.cpp b/document/src/vespa/document/base/forcelink.cpp index b03433081d6..f17c8f582b3 100644 --- a/document/src/vespa/document/base/forcelink.cpp +++ b/document/src/vespa/document/base/forcelink.cpp @@ -3,6 +3,8 @@ #include "forcelink.h" #include <vespa/document/update/updates.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> + namespace document { diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp index 7fa22d27971..ab8736999aa 100644 --- a/document/src/vespa/document/base/globalid.cpp +++ b/document/src/vespa/document/base/globalid.cpp @@ -3,6 +3,7 @@ #include <vespa/document/base/globalid.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_set.hpp> #include <cassert> diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index f5f9f7377b5..4bdecf38996 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "idstring.h" +#include "idstringexception.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/md5.h> +#include <vespa/vespalib/util/stringfmt.h> using vespalib::string; using vespalib::stringref; diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h index f1a96974cc9..5e09b9060b8 100644 --- a/document/src/vespa/document/base/idstring.h +++ b/document/src/vespa/document/base/idstring.h @@ -1,29 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file idstring.h - * - * Contains the various URI schemes accepted in document identifiers. - */ #pragma once -#include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/stllike/string.h> +#include <cstdint> namespace document { /** - * \class document::IdParseException - * \ingroup base - * - * \brief Exception used to indicate failure to parse a %document identifier - * URI. - */ -VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception); - -/** * \class document::IdString * \ingroup base * diff --git a/document/src/vespa/document/base/idstringexception.h b/document/src/vespa/document/base/idstringexception.h new file mode 100644 index 00000000000..922876a8d6e --- /dev/null +++ b/document/src/vespa/document/base/idstringexception.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/exception.h> + +namespace document { + +/** + * \class document::IdParseException + * \ingroup base + * + * \brief Exception used to indicate failure to parse a %document identifier + * URI. + */ + +VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception); + +} diff --git a/document/src/vespa/document/bucket/CMakeLists.txt b/document/src/vespa/document/bucket/CMakeLists.txt index 757884f0050..7dfeaa1920f 100644 --- a/document/src/vespa/document/bucket/CMakeLists.txt +++ b/document/src/vespa/document/bucket/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(document_bucket OBJECT bucketdistribution.cpp bucketid.cpp bucketidfactory.cpp + bucketidlist.cpp bucketselector.cpp bucketspace.cpp DEPENDS diff --git a/document/src/vespa/document/bucket/bucketdistribution.h b/document/src/vespa/document/bucket/bucketdistribution.h index cc7232c81e6..aabc615bdda 100644 --- a/document/src/vespa/document/bucket/bucketdistribution.h +++ b/document/src/vespa/document/bucket/bucketdistribution.h @@ -12,8 +12,8 @@ */ #pragma once +#include "bucketid.h" #include <vector> -#include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/sync.h> namespace document { diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index 8554450fb22..c9efe7cd152 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.hpp> +#include <vespa/vespalib/util/stringfmt.h> using vespalib::nbostream; using vespalib::asciistream; diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h index 63b513dd5a2..79da6c2d11b 100644 --- a/document/src/vespa/document/bucket/bucketid.h +++ b/document/src/vespa/document/bucket/bucketid.h @@ -22,7 +22,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/array.h> namespace vespalib { class nbostream; @@ -33,6 +32,8 @@ namespace document { extern const unsigned char reverseBitTable[256]; +namespace bucket { class BucketIdList; } + class BucketId { public: @@ -47,7 +48,7 @@ public: * typedef when needed we can alter this later with less code changes. */ using Type = uint64_t; - using List = vespalib::Array<BucketId>; + using List = bucket::BucketIdList; /** Create an initially unset bucket id. */ BucketId() : _id(0) {} /** Create a bucket id with the given raw unchecked content. */ diff --git a/document/src/vespa/document/bucket/bucketidlist.cpp b/document/src/vespa/document/bucket/bucketidlist.cpp new file mode 100644 index 00000000000..6c9f7250b8e --- /dev/null +++ b/document/src/vespa/document/bucket/bucketidlist.cpp @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucketidlist.h" + +namespace document::bucket { + +BucketIdList::BucketIdList() { } +BucketIdList::BucketIdList(const BucketIdList & rhs) = default; +BucketIdList & BucketIdList::operator = (const BucketIdList &) = default; +BucketIdList::~BucketIdList() { } + +} diff --git a/document/src/vespa/document/bucket/bucketidlist.h b/document/src/vespa/document/bucket/bucketidlist.h new file mode 100644 index 00000000000..f53301d1b44 --- /dev/null +++ b/document/src/vespa/document/bucket/bucketidlist.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bucketid.h" +#include <vespa/vespalib/util/array.h> + +namespace document::bucket { + +using BucketIdListT = vespalib::Array<BucketId>; + +class BucketIdList : public BucketIdListT { +public: + BucketIdList(); + BucketIdList(BucketIdList && rhs) = default; + BucketIdList & operator = (BucketIdList &&) = default; + BucketIdList(const BucketIdList & rhs); + BucketIdList & operator = (const BucketIdList &); + ~BucketIdList(); +}; + +} diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp index 8b5e7ada91b..7eb3821e283 100644 --- a/document/src/vespa/document/bucket/bucketselector.cpp +++ b/document/src/vespa/document/bucket/bucketselector.cpp @@ -4,10 +4,11 @@ #include "bucketidfactory.h" #include <vespa/document/base/documentid.h> #include <vespa/document/select/node.h> -#include <vespa/document/select/valuenode.h> +#include <vespa/document/select/valuenodes.h> #include <vespa/document/select/visitor.h> #include <vespa/document/select/branch.h> #include <vespa/document/select/compare.h> +#include <vespa/vespalib/util/stringfmt.h> namespace document { diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp index 2591cdbf37c..344fc31ed33 100644 --- a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp +++ b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp @@ -3,6 +3,7 @@ #include "annotationreferencedatatype.h" #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h> #include <ostream> +#include <cassert> using std::unique_ptr; using std::ostream; diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp index 883c4df8f9b..dcf28a9fabf 100644 --- a/document/src/vespa/document/datatype/documenttype.cpp +++ b/document/src/vespa/document/datatype/documenttype.cpp @@ -3,6 +3,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <iomanip> #include <vespa/log/log.h> diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp index 821a22aad4a..afc32e291ea 100644 --- a/document/src/vespa/document/datatype/primitivedatatype.cpp +++ b/document/src/vespa/document/datatype/primitivedatatype.cpp @@ -3,6 +3,7 @@ #include "primitivedatatype.h" #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <sstream> namespace document { diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index 6cf9e18656b..954bd9c742f 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -1,12 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "structdatatype.h" - -#include <iomanip> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/structfieldvalue.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <iomanip> +#include <cassert> + #include <vespa/log/log.h> LOG_SETUP(".document.datatype.struct"); diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp index 4ac09820979..51166adec86 100644 --- a/document/src/vespa/document/fieldset/fieldsets.cpp +++ b/document/src/vespa/document/fieldset/fieldsets.cpp @@ -2,6 +2,7 @@ #include "fieldsets.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> namespace document { diff --git a/document/src/vespa/document/fieldvalue/CMakeLists.txt b/document/src/vespa/document/fieldvalue/CMakeLists.txt index 8c76db54d5f..18ea4fc9368 100644 --- a/document/src/vespa/document/fieldvalue/CMakeLists.txt +++ b/document/src/vespa/document/fieldvalue/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(document_fieldvalues OBJECT fieldvalue.cpp floatfieldvalue.cpp intfieldvalue.cpp + iteratorhandler.cpp literalfieldvalue.cpp longfieldvalue.cpp mapfieldvalue.cpp @@ -22,6 +23,7 @@ vespa_add_library(document_fieldvalues OBJECT structfieldvalue.cpp structuredfieldvalue.cpp tensorfieldvalue.cpp + variablemap.cpp weightedsetfieldvalue.cpp referencefieldvalue.cpp DEPENDS diff --git a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp index 9863ee1c18a..cd6d43d2f4d 100644 --- a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "annotationreferencefieldvalue.h" +#include <vespa/vespalib/util/xmlstream.h> #include <ostream> using std::ostream; @@ -23,8 +24,7 @@ int AnnotationReferenceFieldValue::compare(const FieldValue &other) const { return (getDataType()->getId() - other.getDataType()->getId()); } -void AnnotationReferenceFieldValue::print(ostream &out, bool, - const string &) const { +void AnnotationReferenceFieldValue::print(ostream &out, bool, const string &) const { out << "AnnotationReferenceFieldValue(n)"; } diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp index 866c82a63c9..b5b031d31da 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp @@ -3,14 +3,23 @@ #include "intfieldvalue.h" #include "stringfieldvalue.h" #include "predicatefieldvalue.h" +#include "iteratorhandler.h" #include <vespa/document/util/serializableexceptions.h> +#include <vespa/vespalib/util/polymorphicarrays.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.array"); +using namespace vespalib::xml; + namespace document { using vespalib::IllegalArgumentException; +using fieldvalue::IndexValue; +using fieldvalue::ModificationStatus; +using fieldvalue::IteratorHandler; +using fieldvalue::VariableMap; IMPLEMENT_IDENTIFIABLE_ABSTRACT(ArrayFieldValue, CollectionFieldValue); @@ -23,7 +32,7 @@ ArrayFieldValue::ArrayFieldValue(const DataType &type) "Cannot generate an array value with non-array type " + type.toString() + ".", VESPA_STRLOC); } - _array = createArray(getNestedType()); + _array.reset(static_cast<IArray *>(createArray(getNestedType()).release())); } ArrayFieldValue::ArrayFieldValue(const ArrayFieldValue& other) @@ -175,14 +184,13 @@ ArrayFieldValue::hasChanged() const return false; } -FieldValue::IteratorHandler::ModificationStatus +fieldvalue::ModificationStatus ArrayFieldValue::iterateSubset(int startPos, int endPos, const vespalib::stringref & variable, PathRange nested, - IteratorHandler& handler) const + fieldvalue::IteratorHandler& handler) const { - FieldValue::IteratorHandler::ModificationStatus - retVal = FieldValue::IteratorHandler::NOT_MODIFIED; + fieldvalue::ModificationStatus retVal = ModificationStatus::NOT_MODIFIED; LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')", startPos, endPos, variable.c_str()); @@ -191,16 +199,15 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos, for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) { if (!variable.empty()) { - handler.getVariables()[variable] = IteratorHandler::IndexValue(i); + handler.getVariables()[variable] = IndexValue(i); } - FieldValue::IteratorHandler::ModificationStatus - status = array()[i].iterateNested(nested, handler); + ModificationStatus status = array()[i].iterateNested(nested, handler); - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { indicesToRemove.push_back(i); - retVal = FieldValue::IteratorHandler::MODIFIED; - } else if (status == FieldValue::IteratorHandler::MODIFIED) { + retVal = ModificationStatus::MODIFIED; + } else if (status == ModificationStatus::MODIFIED) { retVal = status; } } @@ -218,7 +225,7 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos, return retVal; } -FieldValue::IteratorHandler::ModificationStatus +fieldvalue::ModificationStatus ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { IteratorHandler::CollectionScope autoScope(handler, *this); @@ -234,7 +241,7 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co case FieldPathEntry::VARIABLE: { LOG(spam, "VARIABLE"); - IteratorHandler::VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); + VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); if (iter != handler.getVariables().end()) { int idx = iter->second.index; @@ -258,17 +265,17 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co } return iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler); } else { - IteratorHandler::ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this)); + ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this)); - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { return status; } if (handler.handleComplex(*this)) { - if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "", - nested, handler) != FieldValue::IteratorHandler::NOT_MODIFIED) + if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler) + != ModificationStatus::NOT_MODIFIED) { - status = FieldValue::IteratorHandler::MODIFIED; + status = ModificationStatus::MODIFIED; } } diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h index 33e07e18588..d61acb8605c 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h @@ -14,25 +14,27 @@ #include "collectionfieldvalue.h" #include <vespa/document/datatype/arraydatatype.h> +#include <vespa/vespalib/util/polymorphicarray.h> namespace document { class ArrayFieldValue : public CollectionFieldValue { private: - IArray::UP _array; + using IArray = vespalib::IArrayT<FieldValue>; + std::unique_ptr<IArray> _array; bool addValue(const FieldValue&) override; bool containsValue(const FieldValue& val) const override; bool removeValue(const FieldValue& val) override; - IteratorHandler::ModificationStatus iterateSubset( + fieldvalue::ModificationStatus iterateSubset( int startPos, int endPos, const vespalib::stringref & variable, PathRange nested, - IteratorHandler& handler) const; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::IteratorHandler& handler) const; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; public: - typedef IArray::const_iterator const_iterator; - typedef IArray::iterator iterator; - typedef std::unique_ptr<ArrayFieldValue> UP; + using const_iterator = IArray::const_iterator; + using iterator = IArray::iterator; + using UP = std::unique_ptr<ArrayFieldValue>; /** * @param arrayType Type of the array. Must be an ArrayDataType, but does diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index f8358cfb544..a8336519e16 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document.h" +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> @@ -8,13 +9,15 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> using vespalib::nbostream; using vespalib::make_string; using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; - +using namespace vespalib::xml; namespace document { namespace { @@ -153,6 +156,11 @@ Document::swap(Document & rhs) std::swap(_lastModified, rhs._lastModified); } +const DocumentType& +Document::getType() const { + return static_cast<const DocumentType &>(StructuredFieldValue::getType()); +} + Document& Document::operator=(const Document& doc) { StructuredFieldValue::operator=(doc); diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h index f159e9be1d5..02b97238cb2 100644 --- a/document/src/vespa/document/fieldvalue/document.h +++ b/document/src/vespa/document/fieldvalue/document.h @@ -18,7 +18,6 @@ #include "structfieldvalue.h" #include <vespa/document/base/documentid.h> #include <vespa/document/base/field.h> -#include <vespa/document/datatype/documenttype.h> namespace document { @@ -71,10 +70,7 @@ public: void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } - const DocumentType& getType() const { - return static_cast<const DocumentType &>(StructuredFieldValue::getType()); - } - + const DocumentType& getType() const; const DocumentId& getId() const { return _id; } DocumentId & getId() { return _id; } diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index fd3093488ea..c137811514c 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -10,17 +10,22 @@ #include "doublefieldvalue.h" #include "bytefieldvalue.h" #include "predicatefieldvalue.h" - +#include "iteratorhandler.h" +#include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/polymorphicarrays.h> +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> using vespalib::FieldBase; using vespalib::nbostream; - +using namespace vespalib::xml; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldValue, vespalib::Identifiable); void FieldValue::serialize(nbostream &stream) const { @@ -51,6 +56,21 @@ FieldValue::hash() const return vespalib::hashValue(os.c_str(), os.size()) ; } +bool +FieldValue::isA(const FieldValue& other) const { + return (getDataType()->isA(*other.getDataType())); +} +int +FieldValue::compare(const FieldValue& other) const { + const DataType & a = *getDataType(); + const DataType & b = *other.getDataType(); + return (a < b) + ? -1 + : (b < a) + ? 1 + : 0; +} + FieldValue& FieldValue::assign(const FieldValue& value) { @@ -168,13 +188,13 @@ FieldValue::onGetNestedFieldValue(PathRange nested) const return FieldValue::UP(); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus FieldValue::iterateNested(PathRange nested, IteratorHandler & handler) const { return onIterateNested(nested, handler); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { if (nested.atEnd()) { @@ -185,78 +205,6 @@ FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const } } -FieldValue::IteratorHandler::~IteratorHandler() { } - -bool -FieldValue::IteratorHandler::IndexValue::operator==(const FieldValue::IteratorHandler::IndexValue& other) const { - if (key.get() != NULL) { - if (other.key.get() != NULL && *key == *other.key) { - return true; - } - return false; - } - - return index == other.index; -} - -FieldValue::IteratorHandler::IndexValue::IndexValue(const FieldValue& key_) - : index(-1), - key(FieldValue::CP(key_.clone())) -{ } - -FieldValue::IteratorHandler::IndexValue::~IndexValue() { } - -vespalib::string -FieldValue::IteratorHandler::IndexValue::toString() const { - if (key.get() != NULL) { - return key->toString(); - } else { - return vespalib::make_string("%d", index); - } -} - -void -FieldValue::IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) { - onPrimitive(fid, Content(fv, getWeight())); -} -bool -FieldValue::IteratorHandler::handleComplex(const FieldValue & fv) { - return onComplex(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleCollectionStart(const FieldValue & fv) { - onCollectionStart(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleCollectionEnd(const FieldValue & fv) { - onCollectionEnd(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleStructStart(const FieldValue & fv) { - onStructStart(Content(fv, getWeight())); -} -void -FieldValue::IteratorHandler::handleStructEnd(const FieldValue & fv) { - onStructEnd(Content(fv, getWeight())); -} - -void -FieldValue::IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) { - (void) fid; - (void) fv; -} - -std::string -FieldValue::IteratorHandler::toString(const VariableMap& vars) { - std::ostringstream out; - out << "[ "; - for (const auto & entry : vars) { - out << entry.first << "=" << entry.second.toString() << " "; - } - out << "]"; - return out.str(); -} - std::string FieldValue::toString(bool verbose, const std::string& indent) const { @@ -280,26 +228,26 @@ private: }; } -FieldValue::IArray::UP +std::unique_ptr<vespalib::IArrayBase> FieldValue::createArray(const DataType & baseType) { switch(baseType.getId()) { case DataType::T_INT: - return IArray::UP(new PrimitiveArrayT<IntFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<IntFieldValue, FieldValue>>(); case DataType::T_FLOAT: - return IArray::UP(new PrimitiveArrayT<FloatFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<FloatFieldValue, FieldValue>>(); case DataType::T_STRING: - return IArray::UP(new PrimitiveArrayT<StringFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<StringFieldValue, FieldValue>>(); case DataType::T_RAW: - return IArray::UP(new PrimitiveArrayT<RawFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<RawFieldValue, FieldValue>>(); case DataType::T_LONG: - return IArray::UP(new PrimitiveArrayT<LongFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<LongFieldValue, FieldValue>>(); case DataType::T_DOUBLE: - return IArray::UP(new PrimitiveArrayT<DoubleFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<DoubleFieldValue, FieldValue>>(); case DataType::T_BYTE: - return IArray::UP(new PrimitiveArrayT<ByteFieldValue, FieldValue>()); + return std::make_unique<PrimitiveArrayT<ByteFieldValue, FieldValue>>(); default: - return IArray::UP(new ComplexArrayT<FieldValue>(FieldValueFactory::UP(new FieldValueFactory(DataType::UP(baseType.clone()))))); + return std::make_unique<ComplexArrayT<FieldValue>>(std::make_unique<FieldValueFactory>(DataType::UP(baseType.clone()))); } } diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index bb2a10a3f53..a96628514c3 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -12,12 +12,12 @@ #pragma once #include "fieldvaluevisitor.h" -#include <vespa/document/datatype/datatype.h> +#include "modificationstatus.h" #include <vespa/document/util/xmlserializable.h> -#include <vespa/vespalib/util/polymorphicarrays.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/document/base/fieldpath.h> #include <vespa/vespalib/objects/cloneable.h> -#include <map> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/util/polymorphicarraybase.h> namespace vespalib { class nbostream; @@ -25,13 +25,19 @@ namespace vespalib { namespace document { +namespace fieldvalue { + class IteratorHandler; +} + +class ByteBuffer; +class DataType; + class FieldValue : public vespalib::Identifiable { protected: FieldValue(const FieldValue&) = default; FieldValue& operator=(const FieldValue&) = default; - using IArray = vespalib::IArrayT<FieldValue>; - static IArray::UP createArray(const DataType & baseType); + static std::unique_ptr<vespalib::IArrayBase> createArray(const DataType & baseType); public: using PathRange = FieldPath::Range<FieldPath::const_iterator>; @@ -39,110 +45,6 @@ public: using SP = std::shared_ptr<FieldValue>; using CP = vespalib::CloneablePtr<FieldValue>; - class IteratorHandler { - public: - class CollectionScope { - public: - CollectionScope(IteratorHandler& handler, const FieldValue& value) - : _handler(handler), _value(value) - { - _handler.handleCollectionStart(_value); - } - - ~CollectionScope() { - _handler.handleCollectionEnd(_value); - } - private: - IteratorHandler& _handler; - const FieldValue& _value; - }; - - class StructScope { - public: - StructScope(IteratorHandler& handler, const FieldValue& value) - : _handler(handler), _value(value) - { - _handler.handleStructStart(_value); - } - - ~StructScope() { - _handler.handleStructEnd(_value); - } - private: - IteratorHandler& _handler; - const FieldValue& _value; - }; - - class IndexValue { - public: - IndexValue() : index(-1), key() {} - IndexValue(int index_) : index(index_), key() {} - IndexValue(const FieldValue& key_); - ~IndexValue(); - - vespalib::string toString() const; - - bool operator==(const IndexValue& other) const; - - int index; // For array - FieldValue::CP key; // For map/wset - }; - - enum ModificationStatus { - MODIFIED, - REMOVED, - NOT_MODIFIED - }; - - typedef std::map<vespalib::string, IndexValue> VariableMap; - protected: - class Content { - public: - Content(const FieldValue & fv, int weight=1) : _fieldValue(fv), _weight(weight) { } - int getWeight() const { return _weight; } - const FieldValue & getValue() const { return _fieldValue; } - private: - const FieldValue & _fieldValue; - int _weight; - }; - IteratorHandler() : _weight(1) { } - public: - virtual ~IteratorHandler(); - - void handlePrimitive(uint32_t fid, const FieldValue & fv); - - /** - Handles a complex type (struct/array/map etc) that is at the end of the - field path. - @return Return true if you want to recurse into the members. - */ - bool handleComplex(const FieldValue& fv); - void handleCollectionStart(const FieldValue & fv); - void handleCollectionEnd(const FieldValue & fv); - void handleStructStart(const FieldValue & fv); - void handleStructEnd(const FieldValue & fv); - void setWeight(int weight) { _weight = weight; } - ModificationStatus modify(FieldValue& fv) { return doModify(fv); } - - VariableMap& getVariables() { return _variables; } - void setVariables(const VariableMap& vars) { _variables = vars; } - static std::string toString(const VariableMap& vars); - virtual bool createMissingPath() const { return false; } - private: - virtual bool onComplex(const Content& fv) { (void) fv; return true; } - virtual void onPrimitive(uint32_t fid, const Content & fv); - virtual void onCollectionStart(const Content & fv) { (void) fv; } - virtual void onCollectionEnd(const Content & fv) { (void) fv; } - virtual void onStructStart(const Content & fv) { (void) fv; } - virtual void onStructEnd(const Content & fv) { (void) fv; } - virtual ModificationStatus doModify(FieldValue&) { return NOT_MODIFIED; }; - - // Scratchpad to store pass on weight. - int getWeight() const { return _weight; } - int _weight; - VariableMap _variables; - }; - DECLARE_IDENTIFIABLE_ABSTRACT(FieldValue); FieldValue() {} @@ -168,27 +70,18 @@ public: virtual const DataType *getDataType() const = 0; /** Wrapper for datatypes isA() function. See DataType. */ - virtual bool isA(const FieldValue& other) const - { return (getDataType()->isA(*other.getDataType())); } + virtual bool isA(const FieldValue& other) const; void serialize(vespalib::nbostream &stream) const; void serialize(ByteBuffer& buffer) const; - ByteBuffer::UP serialize() const; + std::unique_ptr<ByteBuffer> serialize() const; /** * Compares this fieldvalue with another fieldvalue. * Should return 0 if the two are equal, <0 if this object is "less" than * the other, and >0 if this object is more than the other. */ - virtual int compare(const FieldValue& other) const { - const DataType & a = *getDataType(); - const DataType & b = *other.getDataType(); - return (a < b) - ? -1 - : (b < a) - ? 1 - : 0; - } + virtual int compare(const FieldValue& other) const; /** * Returns true if this object have been altered since last @@ -276,9 +169,9 @@ public: * invocations of the before mentioned methods and the additional * onPrimitive. */ - IteratorHandler::ModificationStatus iterateNested(PathRange nested, IteratorHandler & handler) const; + fieldvalue::ModificationStatus iterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const; - IteratorHandler::ModificationStatus iterateNested(const FieldPath& fieldPath, IteratorHandler& handler) const { + fieldvalue::ModificationStatus iterateNested(const FieldPath& fieldPath, fieldvalue::IteratorHandler& handler) const { return iterateNested(fieldPath.begin(), fieldPath.end(), handler); } @@ -292,12 +185,12 @@ public: virtual void printXml(XmlOutputStream& out) const = 0; private: - IteratorHandler::ModificationStatus - iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const { + fieldvalue::ModificationStatus + iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, fieldvalue::IteratorHandler & handler) const { return iterateNested(PathRange(start, end), handler); } virtual FieldValue::UP onGetNestedFieldValue(PathRange nested) const; - virtual IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const; + virtual fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const; }; std::ostream& operator<<(std::ostream& out, const FieldValue & p); diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.cpp b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp new file mode 100644 index 00000000000..394979f31cb --- /dev/null +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include "iteratorhandler.h" + +namespace document::fieldvalue { + +IteratorHandler::~IteratorHandler() { } + + +void +IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) { + onPrimitive(fid, Content(fv, getWeight())); +} +bool +IteratorHandler::handleComplex(const FieldValue & fv) { + return onComplex(Content(fv, getWeight())); +} +void +IteratorHandler::handleCollectionStart(const FieldValue & fv) { + onCollectionStart(Content(fv, getWeight())); +} +void +IteratorHandler::handleCollectionEnd(const FieldValue & fv) { + onCollectionEnd(Content(fv, getWeight())); +} +void +IteratorHandler::handleStructStart(const FieldValue & fv) { + onStructStart(Content(fv, getWeight())); +} +void +IteratorHandler::handleStructEnd(const FieldValue & fv) { + onStructEnd(Content(fv, getWeight())); +} + +void +IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) { + (void) fid; + (void) fv; +} + +} + diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h new file mode 100644 index 00000000000..96290c1ec56 --- /dev/null +++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "variablemap.h" +#include "modificationstatus.h" + +namespace document::fieldvalue { + +class IteratorHandler { +public: + class CollectionScope { + public: + CollectionScope(IteratorHandler &handler, const FieldValue &value) + : _handler(handler), _value(value) { + _handler.handleCollectionStart(_value); + } + + ~CollectionScope() { + _handler.handleCollectionEnd(_value); + } + + private: + IteratorHandler &_handler; + const FieldValue &_value; + }; + + class StructScope { + public: + StructScope(IteratorHandler &handler, const FieldValue &value) + : _handler(handler), _value(value) { + _handler.handleStructStart(_value); + } + + ~StructScope() { + _handler.handleStructEnd(_value); + } + + private: + IteratorHandler &_handler; + const FieldValue &_value; + }; + +protected: + class Content { + public: + Content(const FieldValue &fv, int weight = 1) : _fieldValue(fv), _weight(weight) {} + + int getWeight() const { return _weight; } + + const FieldValue &getValue() const { return _fieldValue; } + + private: + const FieldValue &_fieldValue; + int _weight; + }; + + IteratorHandler() : _weight(1) {} + +public: + virtual ~IteratorHandler(); + void handlePrimitive(uint32_t fid, const FieldValue &fv); + + /** + Handles a complex type (struct/array/map etc) that is at the end of the + field path. + @return Return true if you want to recurse into the members. + */ + bool handleComplex(const FieldValue &fv); + void handleCollectionStart(const FieldValue &fv); + void handleCollectionEnd(const FieldValue &fv); + void handleStructStart(const FieldValue &fv); + void handleStructEnd(const FieldValue &fv); + void setWeight(int weight) { _weight = weight; } + ModificationStatus modify(FieldValue &fv) { return doModify(fv); } + fieldvalue::VariableMap &getVariables() { return _variables; } + void setVariables(const fieldvalue::VariableMap &vars) { _variables = vars; } + virtual bool createMissingPath() const { return false; } +private: + virtual bool onComplex(const Content &fv) { + (void) fv; + return true; + } + + virtual void onPrimitive(uint32_t fid, const Content &fv); + virtual void onCollectionStart(const Content &fv) { (void) fv; } + virtual void onCollectionEnd(const Content &fv) { (void) fv; } + virtual void onStructStart(const Content &fv) { (void) fv; } + virtual void onStructEnd(const Content &fv) { (void) fv; } + virtual ModificationStatus doModify(FieldValue &) { return ModificationStatus::NOT_MODIFIED; }; + + // Scratchpad to store pass on weight. + int getWeight() const { return _weight; } + + int _weight; + fieldvalue::VariableMap _variables; +}; + +} diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp index 4d044219d4c..50c6c901808 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp @@ -1,9 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/literalfieldvalue.h> +#include "literalfieldvalue.h" #include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> +using namespace vespalib::xml; + namespace document { IMPLEMENT_IDENTIFIABLE_ABSTRACT(LiteralFieldValueB, FieldValue); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp index 2186ff28f60..2b37c43be55 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp @@ -2,12 +2,16 @@ #include "mapfieldvalue.h" #include "weightedsetfieldvalue.h" +#include "iteratorhandler.h" #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.map"); using vespalib::Identifiable; +using namespace vespalib::xml; /// \todo TODO (was warning): // Find a way to search through internal map without @@ -15,6 +19,8 @@ using vespalib::Identifiable; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(MapFieldValue, FieldValue); namespace { @@ -31,8 +37,8 @@ const MapDataType *verifyMapType(const DataType& type) { MapFieldValue::MapFieldValue(const DataType &mapType) : FieldValue(), _type(verifyMapType(mapType)), - _keys(createArray(getMapType().getKeyType())), - _values(createArray(getMapType().getValueType())), + _keys(static_cast<IArray *>(createArray(getMapType().getKeyType()).release())), + _values(static_cast<IArray *>(createArray(getMapType().getValueType()).release())), _altered(true) { } @@ -203,8 +209,7 @@ MapFieldValue::compare(const FieldValue& other) const } void -MapFieldValue::print(std::ostream& out, bool verbose, - const std::string& indent) const +MapFieldValue::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "Map("; @@ -268,22 +273,22 @@ MapFieldValue::find(const FieldValue& key) } bool MapFieldValue::checkAndRemove(const FieldValue& key, - FieldValue::IteratorHandler::ModificationStatus status, + ModificationStatus status, bool wasModified, std::vector<const FieldValue*>& keysToRemove) const { - if (status == FieldValue::IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { LOG(spam, "will remove: %s", key.toString().c_str()); keysToRemove.push_back(&key); return true; - } else if (status == FieldValue::IteratorHandler::MODIFIED) { + } else if (status == ModificationStatus::MODIFIED) { return true; } return wasModified; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus MapFieldValue::iterateNestedImpl(PathRange nested, IteratorHandler & handler, const FieldValue& complexFieldValue) const @@ -307,10 +312,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested, wasModified, keysToRemove); } else if (handler.createMissingPath()) { LOG(spam, "creating missing path"); - FieldValue::UP val = - getMapType().getValueType().createFieldValue(); - IteratorHandler::ModificationStatus status = val->iterateNested(nested.next(), handler); - if (status == IteratorHandler::MODIFIED) { + FieldValue::UP val = getMapType().getValueType().createFieldValue(); + ModificationStatus status = val->iterateNested(nested.next(), handler); + if (status == ModificationStatus::MODIFIED) { const_cast<MapFieldValue&>(*this).put(FieldValue::UP(fpe.getLookupKey()->clone()), std::move(val)); return status; } @@ -339,8 +343,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested, case FieldPathEntry::VARIABLE: { LOG(spam, "VARIABLE"); - IteratorHandler::VariableMap::iterator - iter = handler.getVariables().find(fpe.getVariableName()); + VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName()); if (iter != handler.getVariables().end()) { LOG(spam, "variable key = %s", iter->second.key->toString().c_str()); const_iterator found = find(*iter->second.key); @@ -353,12 +356,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested, PathRange next = nested.next(); for (const_iterator it(begin()), mt(end()); it != mt; it++) { LOG(spam, "key is '%s'", it->first->toString().c_str()); - handler.getVariables()[fpe.getVariableName()] - = IteratorHandler::IndexValue(*it->first); - LOG(spam, "vars at this time = %s", - FieldValue::IteratorHandler::toString(handler.getVariables()).c_str()); - wasModified = checkAndRemove(*it->first, - it->second->iterateNested(next, handler), + handler.getVariables()[fpe.getVariableName()] = IndexValue(*it->first); + LOG(spam, "vars at this time = %s", handler.getVariables().toString().c_str()); + wasModified = checkAndRemove(*it->first, it->second->iterateNested(next, handler), wasModified, keysToRemove); } handler.getVariables().erase(fpe.getVariableName()); @@ -371,8 +371,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested, if (isWSet) { handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue()); } - wasModified = checkAndRemove(*it->first, - it->first->iterateNested(nested, handler), + wasModified = checkAndRemove(*it->first, it->first->iterateNested(nested, handler), wasModified, keysToRemove); // Don't iterate over values /*wasModified = checkAndRemove(*it->second, @@ -383,13 +382,12 @@ MapFieldValue::iterateNestedImpl(PathRange nested, } } else { LOG(spam, "at end of field path"); - IteratorHandler::ModificationStatus - status = handler.modify(const_cast<FieldValue&>(complexFieldValue)); + ModificationStatus status = handler.modify(const_cast<FieldValue&>(complexFieldValue)); - if (status == IteratorHandler::REMOVED) { + if (status == ModificationStatus::REMOVED) { LOG(spam, "status = REMOVED"); return status; - } else if (status == IteratorHandler::MODIFIED) { + } else if (status == ModificationStatus::MODIFIED) { LOG(spam, "status = MODIFIED"); wasModified = true; } @@ -419,10 +417,10 @@ MapFieldValue::iterateNestedImpl(PathRange nested, LOG(spam, "erasing map entry with key %s", (*i)->toString().c_str()); const_cast<MapFieldValue&>(*this).erase(**i); } - return wasModified ? IteratorHandler::MODIFIED : IteratorHandler::NOT_MODIFIED; + return wasModified ? ModificationStatus::MODIFIED : ModificationStatus::NOT_MODIFIED; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus MapFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { LOG(spam, "iterating over MapFieldValue"); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.h b/document/src/vespa/document/fieldvalue/mapfieldvalue.h index 8b480cbb7f9..edc20fea777 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.h @@ -26,10 +26,10 @@ private: virtual bool containsValue(const FieldValue& fv) const { return contains(fv); } virtual bool removeValue(const FieldValue& fv) { return erase(fv); } bool checkAndRemove(const FieldValue& key, - FieldValue::IteratorHandler::ModificationStatus status, + fieldvalue::ModificationStatus status, bool wasModified, std::vector<const FieldValue*>& keysToRemove) const; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; // Utility method to avoid constant explicit casting const MapDataType& getMapType() const { return *_type; } @@ -118,8 +118,8 @@ public: void reserve(size_t sz) { _keys->reserve(sz); _values->reserve(sz); } void resize(size_t sz) { _keys->resize(sz); _values->resize(sz); } - IteratorHandler::ModificationStatus iterateNestedImpl(PathRange nested, IteratorHandler & handler, - const FieldValue& complexFieldValue) const; + fieldvalue::ModificationStatus iterateNestedImpl(PathRange nested, fieldvalue::IteratorHandler & handler, + const FieldValue& complexFieldValue) const; // FieldValue implementation FieldValue& assign(const FieldValue&) override; diff --git a/document/src/vespa/document/fieldvalue/modificationstatus.h b/document/src/vespa/document/fieldvalue/modificationstatus.h new file mode 100644 index 00000000000..9ea0c2fa3ae --- /dev/null +++ b/document/src/vespa/document/fieldvalue/modificationstatus.h @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace document::fieldvalue { + +enum class ModificationStatus { + MODIFIED, + REMOVED, + NOT_MODIFIED +}; + +} diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp index 3d39544b3d0..b897a8db930 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "numericfieldvalue.h" +#include <vespa/vespalib/util/xmlstream.h> namespace document { @@ -9,7 +10,7 @@ IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericFieldValueBase, FieldValue); void NumericFieldValueBase::printXml(XmlOutputStream& out) const { - out << XmlContent(getAsString()); + out << vespalib::xml::XmlContent(getAsString()); } diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp index 8e98ebc4c4e..a3394d8fcf4 100644 --- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp @@ -1,13 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "predicatefieldvalue.h" +#include <vespa/document/datatype/datatype.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_printer.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::Slime; using vespalib::slime::SlimeInserter; +using namespace vespalib::xml; namespace document { @@ -67,4 +70,19 @@ void PredicateFieldValue::print(std::ostream& out, bool, const std::string&) con out << PredicatePrinter::print(*_slime) << "\n"; } +const DataType * +PredicateFieldValue::getDataType() const { + return DataType::PREDICATE; +} + +bool +PredicateFieldValue::hasChanged() const { + return _altered; +} + +FieldValue * +PredicateFieldValue::clone() const { + return new PredicateFieldValue(*this); +} + } // namespace document diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h index b38f9e5dcc1..74dc0d11322 100644 --- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h +++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h @@ -2,7 +2,6 @@ #pragma once -#include <memory> #include "fieldvalue.h" namespace vespalib { @@ -25,20 +24,20 @@ public: void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } - virtual FieldValue *clone() const override { return new PredicateFieldValue(*this); } - virtual int compare(const FieldValue &rhs) const override; + FieldValue *clone() const override; + int compare(const FieldValue &rhs) const override; - virtual void printXml(XmlOutputStream &out) const override; - virtual void print(std::ostream &out, bool verbose, const std::string &indent) const override; + void printXml(XmlOutputStream &out) const override; + void print(std::ostream &out, bool verbose, const std::string &indent) const override; - virtual const DataType *getDataType() const override { return DataType::PREDICATE; } - virtual bool hasChanged() const override { return _altered; } + const DataType *getDataType() const override; + bool hasChanged() const override; const vespalib::Slime &getSlime() const { return *_slime; } - virtual FieldValue &assign(const FieldValue &rhs) override; + FieldValue &assign(const FieldValue &rhs) override; -DECLARE_IDENTIFIABLE(PredicateFieldValue); + DECLARE_IDENTIFIABLE(PredicateFieldValue); }; } // namespace document diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp index 96701832b5e..c9950036435 100644 --- a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp @@ -3,6 +3,9 @@ #include "rawfieldvalue.h" #include "literalfieldvalue.hpp" #include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/util/xmlstream.h> + +using namespace vespalib::xml; namespace document { @@ -18,8 +21,7 @@ RawFieldValue::printXml(XmlOutputStream& out) const void RawFieldValue::print(std::ostream& out, bool, const std::string&) const { - StringUtil::printAsHex(out, _value.c_str(), - _value.size()); + StringUtil::printAsHex(out, _value.c_str(), _value.size()); } } // document diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 7a174f1ab52..c8beea5e79c 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -1,16 +1,29 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/document/util/compressor.h> #include <vespa/log/log.h> LOG_SETUP(".document.serializable-array"); - using std::vector; namespace document { +namespace serializablearray { + +using BufferMapT = vespalib::hash_map<int, ByteBuffer::UP>; + +class BufferMap : public BufferMapT { +public: + using BufferMapT::BufferMapT; +}; + +} + SerializableArray::Statistics SerializableArray::_stats; SerializableArray::SerializableArray() @@ -19,6 +32,13 @@ SerializableArray::SerializableArray() { } +serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { + if (!owned) { + owned = std::make_unique<serializablearray::BufferMap>(); + } + return *owned; +} + SerializableArray::SerializableArray(const SerializableArray& other) : Cloneable(), _entries(other._entries), @@ -34,7 +54,7 @@ SerializableArray::SerializableArray(const SerializableArray& other) // Pointing to a buffer in the _owned structure. ByteBuffer::UP buf(ByteBuffer::copyBuffer(e.getBuffer(_uncompSerData.get()), e.size())); e.setBuffer(buf->getBuffer()); - _owned[e.id()] = std::move(buf); + ensure(_owned)[e.id()] = std::move(buf); } else { // If not it is relative to the buffer _uncompSerData, and hence it is valid as is. } @@ -79,7 +99,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer) { maybeDecompress(); Entry e(id, buffer->getRemaining(), buffer->getBuffer()); - _owned[id] = std::move(buffer); + ensure(_owned)[id] = std::move(buffer); EntryMap::iterator it = find(id); if (it == _entries.end()) { _entries.push_back(e); @@ -149,7 +169,9 @@ SerializableArray::clear(int id) EntryMap::iterator it = find(id); if (it != _entries.end()) { _entries.erase(it); - _owned.erase(id); + if (_owned) { + _owned->erase(id); + } invalidate(); } } @@ -215,4 +237,14 @@ void SerializableArray::assign(EntryMap & entries, } } +CompressionInfo +SerializableArray::getCompressionInfo() const { + return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); +} + +const char * +SerializableArray::Entry::getBuffer(const ByteBuffer * readOnlyBuffer) const { + return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); +} + } // document diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 9c0978f1476..1d211a50ab3 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,20 +16,22 @@ #pragma once -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> -#include <vespa/document/util/serializable.h> -#include <vector> #include <vespa/vespalib/objects/cloneable.h> -#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/buffer.h> -#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/vespalib/util/memory.h> +#include <vector> -namespace document -{ +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) + +namespace document { class SerializableArrayIterator; +class ByteBuffer; + +namespace serializablearray { + class BufferMap; +} class SerializableArray : public vespalib::Cloneable { @@ -71,7 +73,7 @@ public: bool operator < (const Entry & e) const { return cmp(e) < 0; } int cmp(const Entry & e) const { return _id - e._id; } void setBuffer(const char * buffer) { _data._buffer = buffer; _sz |= BUFFER_MASK; } - const char * getBuffer(const ByteBuffer * readOnlyBuffer) const { return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); } + VESPA_DLL_LOCAL const char * getBuffer(const ByteBuffer * readOnlyBuffer) const; private: uint32_t getOffset() const { return _data._offset; } enum { BUFFER_MASK=0x80000000 }; @@ -100,12 +102,11 @@ public: private: static Statistics _stats; - typedef vespalib::hash_map<int, uint32_t> HashMap; - public: static Statistics& getStatistics() { return _stats; } - typedef vespalib::CloneablePtr<SerializableArray> CP; - typedef std::unique_ptr<SerializableArray> UP; + using CP = vespalib::CloneablePtr<SerializableArray>; + using UP = std::unique_ptr<SerializableArray>; + using ByteBufferUP = std::unique_ptr<ByteBuffer>; SerializableArray(); virtual ~SerializableArray(); @@ -122,7 +123,7 @@ public: void set(int id, const char* value, int len); /** Stores a value in the array. */ - void set(int id, std::unique_ptr<ByteBuffer> buffer); + void set(int id, ByteBufferUP buffer); /** * Gets a value from the array. This is the faster version of the above. @@ -152,19 +153,17 @@ public: void clear(); CompressionConfig::Type getCompression() const { return _serializedCompression; } - CompressionInfo getCompressionInfo() const { - return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); - } + CompressionInfo getCompressionInfo() const; /** * Sets the serialized data that is the basis for this object's * content. This is used by deserialization. Any existing entries * are cleared. */ - VESPA_DLL_LOCAL void assign(EntryMap &entries, - ByteBuffer::UP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length); + void assign(EntryMap &entries, + ByteBufferUP buffer, + CompressionConfig::Type comp_type, + uint32_t uncompressed_length); bool empty() const { return _entries.empty(); } @@ -189,22 +188,22 @@ private: return false; } - VESPA_DLL_LOCAL bool deCompressAndCatch() const; + bool deCompressAndCatch() const; void maybeDecompress() const { if ( shouldDecompress() ) { const_cast<SerializableArray *>(this)->deCompress(); } } - VESPA_DLL_LOCAL void deCompress(); // throw (DeserializeException); + void deCompress(); // throw (DeserializeException); /** Contains the stored attributes, with reference to the real data.. */ EntryMap _entries; /** The buffers we own. */ - vespalib::hash_map<int, ByteBuffer::UP > _owned; + std::unique_ptr<serializablearray::BufferMap> _owned; /** Data we deserialized from, if applicable. */ - ByteBuffer::UP _uncompSerData; - ByteBuffer::UP _compSerData; + ByteBufferUP _uncompSerData; + ByteBufferUP _compSerData; CompressionConfig::Type _serializedCompression; uint32_t _uncompressedLength; @@ -215,4 +214,3 @@ private: }; } // document - diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 079b730c8cc..cdfba106c49 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -8,9 +8,10 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/datatype/positiondatatype.h> -#include <vespa/vespalib/util/vstringfmt.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".document.structfieldvalue"); @@ -19,6 +20,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using namespace vespalib::xml; namespace document { @@ -43,6 +45,26 @@ StructFieldValue::~StructFieldValue() { } StructFieldValue::Chunks::~Chunks() { } void +StructFieldValue::Chunks::push_back(SerializableArray::UP item) { + assert(_sz < 2); + _chunks[_sz++].reset(item.release()); +} + +void +StructFieldValue::Chunks::clear() { + _chunks[0].reset(); + _chunks[1].reset(); + _sz = 0; +} + +void +StructFieldValue::Chunks::swap(Chunks & rhs) { + _chunks[0].swap(rhs._chunks[0]); + _chunks[1].swap(rhs._chunks[1]); + std::swap(_sz, rhs._sz); +} + +void StructFieldValue::swap(StructFieldValue & rhs) { StructuredFieldValue::swap(rhs); @@ -53,6 +75,16 @@ StructFieldValue::swap(StructFieldValue & rhs) std::swap(_version, _version); } +const StructDataType & +StructFieldValue::getStructType() const { + return static_cast<const StructDataType &>(getType()); +} + +const CompressionConfig & +StructFieldValue::getCompressionConfig() const { + return getStructType().getCompressionConfig(); +} + void StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index a46a36c477d..01a0e732e43 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -11,17 +11,16 @@ #include "structuredfieldvalue.h" #include "serializablearray.h" -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/datatype/structdatatype.h> -#include <vector> namespace document { + class Document; class DocumentType; class DocumentTypeRepo; class FieldValueWriter; class FixedTypeRepo; class FieldSet; +class StructDataType; class StructFieldValue : public StructuredFieldValue { @@ -32,24 +31,13 @@ public: ~Chunks(); SerializableArray & operator [] (size_t i) { return *_chunks[i]; } const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; } - void push_back(SerializableArray::UP item) { - assert(_sz < 2); - _chunks[_sz++].reset(item.release()); - } + VESPA_DLL_LOCAL void push_back(SerializableArray::UP item); SerializableArray & back() { return *_chunks[_sz-1]; } const SerializableArray & back() const { return *_chunks[_sz-1]; } size_t size() const { return _sz; } bool empty() const { return _sz == 0; } - void clear() { - _chunks[0].reset(); - _chunks[1].reset(); - _sz = 0; - } - void swap(Chunks & rhs) { - _chunks[0].swap(rhs._chunks[0]); - _chunks[1].swap(rhs._chunks[1]); - std::swap(_sz, rhs._sz); - } + VESPA_DLL_LOCAL void clear(); + VESPA_DLL_LOCAL void swap(Chunks & rhs); private: SerializableArray::CP _chunks[2]; size_t _sz; @@ -73,12 +61,12 @@ public: const DocumentTypeRepo * getRepo() const { return _repo; } void setDocumentType(const DocumentType & docType) { _doc_type = & docType; } - const StructDataType & getStructType() const { return static_cast<const StructDataType &>(getType()); } + const StructDataType & getStructType() const; void lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fields, - ByteBuffer::UP buffer, + std::unique_ptr<ByteBuffer> buffer, CompressionConfig::Type comp_type, int32_t uncompressed_length); @@ -99,8 +87,7 @@ public: const Field& getField(const vespalib::stringref & name) const override; void clear() override; - const CompressionConfig &getCompressionConfig() const - { return getStructType().getCompressionConfig(); } + const CompressionConfig &getCompressionConfig() const; // FieldValue implementation. FieldValue& assign(const FieldValue&) override; @@ -146,4 +133,3 @@ private: }; } // document - diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp index 6d77d101ff2..8e30cba2052 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp @@ -1,13 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "structuredfieldvalue.hpp" -#include "fieldvalues.h" +#include "iteratorhandler.h" +#include "weightedsetfieldvalue.h" +#include "arrayfieldvalue.h" #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.structured"); +using vespalib::IllegalArgumentException; + namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructuredFieldValue, FieldValue); StructuredFieldValue::Iterator::Iterator() @@ -59,7 +65,7 @@ void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & if (!field.getDataType().isValueType(value) && !value.getDataType()->isA(field.getDataType())) { - throw vespalib::IllegalArgumentException( + throw IllegalArgumentException( "Cannot assign value of type " + value.getDataType()->toString() + "with value : '" + value.toString() + "' to field " + field.getName().c_str() + " of type " @@ -81,7 +87,7 @@ StructuredFieldValue::onGetNestedFieldValue(PathRange nested) const return fv; } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { IteratorHandler::StructScope autoScope(handler, *this); @@ -93,38 +99,35 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle LOG(spam, "fieldRef = %s", fpe.getFieldRef().toString().c_str()); LOG(spam, "fieldValueToSet = %s", fpe.getFieldValueToSet().toString().c_str()); if (exists) { - IteratorHandler::ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); - if (status == IteratorHandler::REMOVED) { + ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); + if (status == ModificationStatus::REMOVED) { LOG(spam, "field exists, status = REMOVED"); const_cast<StructuredFieldValue&>(*this).remove(fpe.getFieldRef()); - return IteratorHandler::MODIFIED; - } else if (status == IteratorHandler::MODIFIED) { + return ModificationStatus::MODIFIED; + } else if (status == ModificationStatus::MODIFIED) { LOG(spam, "field exists, status = MODIFIED"); const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet()); - return IteratorHandler::MODIFIED; + return ModificationStatus::MODIFIED; } else { - LOG(spam, "field exists, status = %u", status); return status; } } else if (handler.createMissingPath()) { LOG(spam, "createMissingPath is true"); - IteratorHandler::ModificationStatus status - = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); - if (status == IteratorHandler::MODIFIED) { + ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler); + if (status == ModificationStatus::MODIFIED) { LOG(spam, "field did not exist, status = MODIFIED"); const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet()); return status; } } LOG(spam, "field did not exist, returning NOT_MODIFIED"); - return IteratorHandler::NOT_MODIFIED; + return ModificationStatus::NOT_MODIFIED; } else { - throw vespalib::IllegalArgumentException("Illegal field path for struct value"); + throw IllegalArgumentException("Illegal field path for struct value"); } } else { - IteratorHandler::ModificationStatus - status = handler.modify(const_cast<StructuredFieldValue&>(*this)); - if (status == IteratorHandler::REMOVED) { + ModificationStatus status = handler.modify(const_cast<StructuredFieldValue&>(*this)); + if (status == ModificationStatus::REMOVED) { LOG(spam, "field REMOVED"); return status; } @@ -133,11 +136,11 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle LOG(spam, "handleComplex"); std::vector<const Field*> fieldsToRemove; for (const_iterator it(begin()), mt(end()); it != mt; ++it) { - IteratorHandler::ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler); - if (currStatus == IteratorHandler::REMOVED) { + ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler); + if (currStatus == ModificationStatus::REMOVED) { fieldsToRemove.push_back(&it.field()); - status = IteratorHandler::MODIFIED; - } else if (currStatus == IteratorHandler::MODIFIED) { + status = ModificationStatus::MODIFIED; + } else if (currStatus == ModificationStatus::MODIFIED) { status = currStatus; } } diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h index 4cccbf05de0..eb5912734c5 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h @@ -92,8 +92,8 @@ protected: virtual void setFieldValue(const Field&, FieldValue::UP value) = 0; void setFieldValue(const Field & field, const FieldValue & value); - IteratorHandler::ModificationStatus - onIterateNested(PathRange nested, IteratorHandler & handler) const override; + fieldvalue::ModificationStatus + onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override; public: DECLARE_IDENTIFIABLE_ABSTRACT(StructuredFieldValue); diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp index d6396c43a46..d7d7721e833 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp @@ -1,10 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensorfieldvalue.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/eval/tensor/tensor.h> #include <ostream> +#include <cassert> using vespalib::tensor::Tensor; +using namespace vespalib::xml; namespace document { @@ -164,7 +168,6 @@ TensorFieldValue::compare(const FieldValue &other) const return ((_tensor.get() < rhs._tensor.get()) ? -1 : 1); } - IMPLEMENT_IDENTIFIABLE(TensorFieldValue, FieldValue); } // document diff --git a/document/src/vespa/document/fieldvalue/variablemap.cpp b/document/src/vespa/document/fieldvalue/variablemap.cpp new file mode 100644 index 00000000000..b38b5baeec2 --- /dev/null +++ b/document/src/vespa/document/fieldvalue/variablemap.cpp @@ -0,0 +1,59 @@ +#include "variablemap.h" +#include "fieldvalue.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace document::fieldvalue { + +IndexValue::IndexValue() : index(-1), key() {} +IndexValue::IndexValue(int index_) : index(index_), key() {} + +bool +IndexValue::operator==(const IndexValue& other) const { + if (key.get() != NULL) { + if (other.key.get() != NULL && *key == *other.key) { + return true; + } + return false; + } + + return index == other.index; +} + +IndexValue::IndexValue(const FieldValue& key_) + : index(-1), + key(FieldValue::CP(key_.clone())) +{ } + +IndexValue::IndexValue(const IndexValue & rhs) = default; +IndexValue & IndexValue::operator = (const IndexValue & rhs) = default; + +IndexValue::~IndexValue() { } + +vespalib::string +IndexValue::toString() const { + if (key.get() != NULL) { + return key->toString(); + } else { + return vespalib::make_string("%d", index); + } +} + +VariableMap::VariableMap() {} +VariableMap::~VariableMap() {} +VariableMap::VariableMap(const VariableMap & rhs) = default; +VariableMap & VariableMap::operator = (const VariableMap & rhs) = default; + +vespalib::string +VariableMap::toString() const { + vespalib::asciistream out; + out << "[ "; + for (const auto & entry : *this) { + out << entry.first << "=" << entry.second.toString() << " "; + } + out << "]"; + return out.str(); +} + +} + diff --git a/document/src/vespa/document/fieldvalue/variablemap.h b/document/src/vespa/document/fieldvalue/variablemap.h new file mode 100644 index 00000000000..06679682e0c --- /dev/null +++ b/document/src/vespa/document/fieldvalue/variablemap.h @@ -0,0 +1,47 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/memory.h> +#include <vespa/vespalib/stllike/string.h> +#include <map> + +namespace document { + class FieldValue; +} + +namespace document::fieldvalue { + +class IndexValue { +public: + IndexValue(); + IndexValue(int index_); + IndexValue(const FieldValue& key_); + IndexValue(IndexValue && rhs) = default; + IndexValue & operator = (IndexValue && rhs) = default; + IndexValue(const IndexValue & rhs); + IndexValue & operator = (const IndexValue & rhs); + + ~IndexValue(); + + vespalib::string toString() const; + bool operator==(const IndexValue& other) const; + + int index; // For array + vespalib::CloneablePtr<FieldValue> key; // For map/wset +}; + +using VariableMapT = std::map<vespalib::string, IndexValue>; + +class VariableMap : public VariableMapT { +public: + VariableMap(); + VariableMap(VariableMap && rhs) = default; + VariableMap & operator = (VariableMap && rhs) = default; + VariableMap(const VariableMap & rhs); + VariableMap & operator = (const VariableMap & rhs); + ~VariableMap(); + vespalib::string toString() const; +}; + +}
\ No newline at end of file diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp index 7bf5f5350d3..67e0c8327e6 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp @@ -2,14 +2,20 @@ #include "weightedsetfieldvalue.h" #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> #include <ostream> using vespalib::Identifiable; +using vespalib::IllegalArgumentException; +using vespalib::IllegalStateException; +using namespace vespalib::xml; /// \todo TODO (was warning): Find a way to search through internal map without duplicating keys to create shared pointers. namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue, CollectionFieldValue); namespace { @@ -17,9 +23,8 @@ const DataType &getKeyType(const DataType &type) { const WeightedSetDataType *wtype = Identifiable::cast<const WeightedSetDataType *>(&type); if (!wtype) { - throw vespalib::IllegalArgumentException( - "Cannot generate a weighted set value with non-weighted set " - "type " + type.toString() + ".", VESPA_STRLOC); + throw IllegalArgumentException("Cannot generate a weighted set value with non-weighted set " + "type " + type.toString() + ".", VESPA_STRLOC); } return wtype->getNestedType(); } @@ -39,8 +44,7 @@ WeightedSetFieldValue::~WeightedSetFieldValue() { } void WeightedSetFieldValue::verifyKey(const FieldValue & v) { if (!getNestedType().isValueType(v)) { - throw InvalidDataTypeException(*v.getDataType(), getNestedType(), - VESPA_STRLOC); + throw InvalidDataTypeException(*v.getDataType(), getNestedType(), VESPA_STRLOC); } } @@ -92,9 +96,7 @@ WeightedSetFieldValue::increment(const FieldValue& key, int val) } } else { if (it == _map.end()) { - throw vespalib::IllegalStateException("Cannot modify non-existing " - "entry in weightedset without createIfNonExistent set", - VESPA_STRLOC); + throw IllegalStateException("Cannot modify non-existing entry in weightedset without createIfNonExistent set", VESPA_STRLOC); } IntFieldValue& fv = static_cast<IntFieldValue&>(*it->second); fv.setValue(fv.getValue() + val); @@ -203,7 +205,7 @@ WeightedSetFieldValue::find(const FieldValue& key) return _map.find(key); } -FieldValue::IteratorHandler::ModificationStatus +ModificationStatus WeightedSetFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const { return _map.iterateNestedImpl(nested, handler, *this); diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h index 4d6be1fd35a..5de0bf91614 100644 --- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h @@ -7,9 +7,9 @@ */ #pragma once -#include <vespa/document/fieldvalue/collectionfieldvalue.h> -#include <vespa/document/fieldvalue/mapfieldvalue.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> +#include "collectionfieldvalue.h" +#include "mapfieldvalue.h" +#include "intfieldvalue.h" #include <vespa/document/datatype/weightedsetdatatype.h> #include <map> @@ -33,7 +33,7 @@ private: bool addValue(const FieldValue& fval) override { return add(fval, 1); } bool containsValue(const FieldValue& val) const override; bool removeValue(const FieldValue& val) override; - IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler& handler) const override; + fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler& handler) const override; public: typedef std::unique_ptr<WeightedSetFieldValue> UP; diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 78d668a4886..93adce76efd 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -3,7 +3,6 @@ #include "documenttyperepo.h" #include <vespa/document/datatype/annotationreferencedatatype.h> -#include <vespa/document/datatype/annotationtype.h> #include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/mapdatatype.h> @@ -11,15 +10,10 @@ #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/datatype/referencedatatype.h> -#include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/document/config/config-documenttypes.h> #include <fstream> -#include <memory> -#include <utility> #include <vespa/log/log.h> LOG_SETUP(".documenttyperepo"); @@ -39,6 +33,20 @@ using vespalib::stringref; namespace document { +namespace internal { + +using DocumentTypeMapT = vespalib::hash_map<int32_t, DataTypeRepo *>; + +class DocumentTypeMap : public DocumentTypeMapT +{ +public: + using DocumentTypeMapT::DocumentTypeMapT; +}; + +} + +using DocumentTypeMap = internal::DocumentTypeMap; + namespace { template <typename Container> void DeleteContent(Container &c) { @@ -359,7 +367,6 @@ void addDataTypes(const vector<Datatype> &types, Repo &repo, } } -typedef hash_map<int32_t, DataTypeRepo *> DocumentTypeMap; void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) { for (DocumentTypeMap::const_iterator it = type_map.begin(); it != type_map.end(); ++it) { @@ -515,49 +522,55 @@ void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t, } // namespace -DocumentTypeRepo::DocumentTypeRepo() { - addDefaultDocument(_doc_types); +DocumentTypeRepo::DocumentTypeRepo() : + _doc_types(std::make_unique<internal::DocumentTypeMap>()) +{ + addDefaultDocument(*_doc_types); } -DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) { - addDefaultDocument(_doc_types); +DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) : + _doc_types(std::make_unique<internal::DocumentTypeMap>()) +{ + addDefaultDocument(*_doc_types); try { - addDataTypeRepo(makeDataTypeRepo(type, _doc_types), _doc_types); + addDataTypeRepo(makeDataTypeRepo(type, *_doc_types), *_doc_types); } catch (...) { - DeleteMapContent(_doc_types); + DeleteMapContent(*_doc_types); throw; } } -DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) { - addDefaultDocument(_doc_types); +DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) : + _doc_types(std::make_unique<internal::DocumentTypeMap>()) +{ + addDefaultDocument(*_doc_types); try { - createAllDocumentTypes(config.documenttype, _doc_types); - addAllDocumentTypesToRepos(_doc_types); - configureAllRepos(config.documenttype, _doc_types); + createAllDocumentTypes(config.documenttype, *_doc_types); + addAllDocumentTypesToRepos(*_doc_types); + configureAllRepos(config.documenttype, *_doc_types); } catch (...) { - DeleteMapContent(_doc_types); + DeleteMapContent(*_doc_types); throw; } } DocumentTypeRepo::~DocumentTypeRepo() { - DeleteMapContent(_doc_types); + DeleteMapContent(*_doc_types); } const DocumentType *DocumentTypeRepo::getDocumentType(int32_t type_id) const { - const DataTypeRepo *repo = FindPtr(_doc_types, type_id); + const DataTypeRepo *repo = FindPtr(*_doc_types, type_id); return repo ? repo->doc_type : nullptr; } const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) const { DocumentTypeMap::const_iterator it = - _doc_types.find(DocumentType::createId(name)); + _doc_types->find(DocumentType::createId(name)); - if (it != _doc_types.end() && it->second->doc_type->getName() == name) { + if (it != _doc_types->end() && it->second->doc_type->getName() == name) { return it->second->doc_type; } - for (it = _doc_types.begin(); it != _doc_types.end(); ++it) { + for (it = _doc_types->begin(); it != _doc_types->end(); ++it) { if (it->second->doc_type->getName() == name) { return it->second->doc_type; } @@ -567,27 +580,27 @@ const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) con const DataType * DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const { - const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); return dt_repo ? dt_repo->repo.lookup(id) : nullptr; } const DataType * DocumentTypeRepo::getDataType( const DocumentType &doc_type, const stringref &name) const { - const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); return dt_repo ? dt_repo->repo.lookup(name) : nullptr; } const AnnotationType *DocumentTypeRepo::getAnnotationType( const DocumentType &doc_type, int32_t id) const { - const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId()); + const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId()); return dt_repo ? dt_repo->annotations.lookup(id) : nullptr; } void DocumentTypeRepo::forEachDocumentType( Closure1<const DocumentType &> &c) const { for (DocumentTypeMap::const_iterator - it = _doc_types.begin(); it != _doc_types.end(); ++it) { + it = _doc_types->begin(); it != _doc_types->end(); ++it) { c.call(*it->second->doc_type); } } diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h index 55655fe169e..4955f3562db 100644 --- a/document/src/vespa/document/repo/documenttyperepo.h +++ b/document/src/vespa/document/repo/documenttyperepo.h @@ -3,7 +3,6 @@ #pragma once #include <memory> -#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/closure.h> @@ -11,6 +10,7 @@ namespace document { namespace internal { class InternalDocumenttypesType; + class DocumentTypeMap; } class AnnotationType; @@ -19,9 +19,7 @@ class DataTypeRepo; class DocumentType; class DocumentTypeRepo { - typedef vespalib::hash_map<int32_t, DataTypeRepo *> DocumentTypeMap; - - DocumentTypeMap _doc_types; + std::unique_ptr<internal::DocumentTypeMap> _doc_types; public: using DocumenttypesConfig = const internal::InternalDocumenttypesType; diff --git a/document/src/vespa/document/select/.gitignore b/document/src/vespa/document/select/.gitignore index 69e6eac20bb..5f004816692 100644 --- a/document/src/vespa/document/select/.gitignore +++ b/document/src/vespa/document/select/.gitignore @@ -2,4 +2,3 @@ Makefile .depend* .*.swp *.So - diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index 8521c525171..5077c9f0494 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -1,26 +1,27 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(document_select OBJECT SOURCES + bodyfielddetector.cpp branch.cpp + cloningvisitor.cpp compare.cpp constant.cpp + context.cpp doctype.cpp + gid_filter.cpp + invalidconstant.cpp operator.cpp - parser.cpp - result.cpp - value.cpp - valuenode.cpp orderingselector.cpp orderingspecification.cpp + parser.cpp + result.cpp + resultset.cpp resultlist.cpp - invalidconstant.cpp - bodyfielddetector.cpp simpleparser.cpp - resultset.cpp traversingvisitor.cpp - cloningvisitor.cpp - context.cpp - gid_filter.cpp + value.cpp + valuenode.cpp + valuenodes.cpp AFTER document_documentconfig ) diff --git a/document/src/vespa/document/select/bodyfielddetector.cpp b/document/src/vespa/document/select/bodyfielddetector.cpp index d0ddd0043ef..2b4c9890e45 100644 --- a/document/src/vespa/document/select/bodyfielddetector.cpp +++ b/document/src/vespa/document/select/bodyfielddetector.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bodyfielddetector.h" +#include "valuenodes.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/vespalib/util/closure.h> -#include "valuenode.h" -namespace document { - -namespace select { +namespace document::select { void BodyFieldDetector::detectFieldType(const FieldValueNode *expr, @@ -35,10 +32,8 @@ BodyFieldDetector::detectFieldType(const FieldValueNode *expr, void BodyFieldDetector::visitFieldValueNode(const FieldValueNode& expr) { - _repo.forEachDocumentType( - *makeClosure(this, &BodyFieldDetector::detectFieldType, &expr)); + _repo.forEachDocumentType(*makeClosure(this, &BodyFieldDetector::detectFieldType, &expr)); } -} // namespace select -} // namespace document +} diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp index ddbad69b14d..ea0d0ef5646 100644 --- a/document/src/vespa/document/select/branch.cpp +++ b/document/src/vespa/document/select/branch.cpp @@ -2,6 +2,8 @@ #include "branch.h" #include "visitor.h" +#include <cassert> +#include <ostream> namespace document { namespace select { diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp index 84188788416..c0a45b758d5 100644 --- a/document/src/vespa/document/select/cloningvisitor.cpp +++ b/document/src/vespa/document/select/cloningvisitor.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cloningvisitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "branch.h" #include "compare.h" #include "constant.h" diff --git a/document/src/vespa/document/select/constant.cpp b/document/src/vespa/document/select/constant.cpp index 686ab05055c..0e2fdc2ecec 100644 --- a/document/src/vespa/document/select/constant.cpp +++ b/document/src/vespa/document/select/constant.cpp @@ -2,9 +2,10 @@ #include "constant.h" #include "visitor.h" +#include <cassert> +#include <ostream> -namespace document { -namespace select { +namespace document::select { Constant::Constant(const vespalib::stringref & value) : Node(value), @@ -46,13 +47,11 @@ Constant::visit(Visitor &v) const void -Constant::print(std::ostream& out, bool, - const std::string&) const +Constant::print(std::ostream& out, bool, const std::string&) const { if (_parentheses) out << '('; out << _name; if (_parentheses) out << ')'; } -} // select -} // document +} diff --git a/document/src/vespa/document/select/context.cpp b/document/src/vespa/document/select/context.cpp index 44d005e9db2..ef29770ddc8 100644 --- a/document/src/vespa/document/select/context.cpp +++ b/document/src/vespa/document/select/context.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "context.h" +#include "variablemap.h" +#include <vespa/document/select/value.h> -namespace document { -namespace select { +namespace document::select { -Context::Context(void) +Context::Context() : _doc(NULL), _docId(NULL), _docUpdate(NULL), @@ -35,5 +36,20 @@ Context::Context(const DocumentUpdate& docUpdate) Context::~Context() { } -} // select -} // document +std::unique_ptr<Value> +Context::getValue(const vespalib::string & value) const { + VariableMap::const_iterator iter = _variables->find(value); + + if (iter != _variables->end()) { + return std::make_unique<FloatValue>(iter->second); + } else { + return std::make_unique<FloatValue>(0.0); + } +} + +void +Context::setVariableMap(std::unique_ptr<VariableMap> map) { + _variables = std::move(map); +} + +} diff --git a/document/src/vespa/document/select/context.h b/document/src/vespa/document/select/context.h index 11a3a2be51c..0e347e22bb5 100644 --- a/document/src/vespa/document/select/context.h +++ b/document/src/vespa/document/select/context.h @@ -2,9 +2,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_map.h> +#include <memory> -namespace document{ +namespace document { class Document; class DocumentId; @@ -12,24 +12,26 @@ class DocumentUpdate; namespace select { -class Context -{ -public: - typedef vespalib::hash_map<vespalib::string, double> VariableMap; +class Value; +class VariableMap; +class Context { +public: Context(); - Context(const Document& doc); - Context(const DocumentId& docId); - Context(const DocumentUpdate& docUpdate); + Context(const Document & doc); + Context(const DocumentId & docId); + Context(const DocumentUpdate & docUpdate); virtual ~Context(); - const Document * _doc; - const DocumentId * _docId; - const DocumentUpdate * _docUpdate; - VariableMap _variables; -}; - -} // select -} // document + void setVariableMap(std::unique_ptr<VariableMap> map); + std::unique_ptr<Value> getValue(const vespalib::string & value) const; + const Document *_doc; + const DocumentId *_docId; + const DocumentUpdate *_docUpdate; +private: + std::unique_ptr<VariableMap> _variables; +}; +} +} diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index 3c23b2e5ba1..117f97ea1bc 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -5,9 +5,9 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> -namespace document { -namespace select { +namespace document::select { namespace { bool documentTypeEqualsName(const DocumentType& type, @@ -88,5 +88,4 @@ DocType::print(std::ostream& out, bool verbose, if (_parentheses) out << ')'; } -} // select -} // document +} diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp index cb194452ead..7630053c16f 100644 --- a/document/src/vespa/document/select/gid_filter.cpp +++ b/document/src/vespa/document/select/gid_filter.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "gid_filter.h" #include "node.h" -#include "parser.h" #include "visitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "compare.h" #include "branch.h" #include <vespa/document/base/idstring.h> -namespace document { -namespace select { +namespace document::select { namespace { @@ -156,5 +154,4 @@ GidFilter::GidFilter(const Node& ast_root) { } -} // select -} // document +} diff --git a/document/src/vespa/document/select/invalidconstant.cpp b/document/src/vespa/document/select/invalidconstant.cpp index 793e34c7bc5..53e3a6e1647 100644 --- a/document/src/vespa/document/select/invalidconstant.cpp +++ b/document/src/vespa/document/select/invalidconstant.cpp @@ -2,9 +2,9 @@ #include "invalidconstant.h" #include "visitor.h" +#include <ostream> -namespace document { -namespace select { +namespace document::select { InvalidConstant::InvalidConstant(const vespalib::stringref & value) : Node(value) @@ -26,13 +26,11 @@ InvalidConstant::visit(Visitor &v) const void -InvalidConstant::print(std::ostream& out, bool, - const std::string&) const +InvalidConstant::print(std::ostream& out, bool, const std::string&) const { if (_parentheses) out << '('; out << _name; if (_parentheses) out << ')'; } -} // select -} // document +} diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h index c56203da76c..6ccc1fa5dec 100644 --- a/document/src/vespa/document/select/node.h +++ b/document/src/vespa/document/select/node.h @@ -12,7 +12,6 @@ #pragma once -#include <string> #include "resultlist.h" #include "context.h" @@ -33,15 +32,11 @@ public: typedef std::shared_ptr<Node> SP; Node(const vespalib::stringref & name) : _name(name), _parentheses(false) {} - virtual ~Node() {} + ~Node() override {} void setParentheses() { _parentheses = true; } - void - clearParentheses() - { - _parentheses = false; - } + void clearParentheses() { _parentheses = false; } bool hadParentheses() const { return _parentheses; } diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index b127e29c1ad..3c09a482cf0 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -4,9 +4,9 @@ #include <vespa/vespalib/util/regexp.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <cassert> -namespace document { -namespace select { +namespace document::select { Operator::OperatorMap Operator::_operators; @@ -228,5 +228,4 @@ GlobOperator::containsVariables(const vespalib::stringref & expression) const GlobOperator GlobOperator::GLOB("="); -} // select -} // document +} diff --git a/document/src/vespa/document/select/orderingselector.cpp b/document/src/vespa/document/select/orderingselector.cpp index 32a1f0a8adf..4195a6c28a2 100644 --- a/document/src/vespa/document/select/orderingselector.cpp +++ b/document/src/vespa/document/select/orderingselector.cpp @@ -2,18 +2,16 @@ #include "orderingselector.h" #include "node.h" -#include "valuenode.h" +#include "valuenodes.h" #include "visitor.h" #include "compare.h" #include "branch.h" -#include <algorithm> #include <vespa/document/base/documentid.h> -#include <vespa/document/base/idstring.h> namespace document { -using namespace document::select; +using namespace select; namespace { /** diff --git a/document/src/vespa/document/select/parser.h b/document/src/vespa/document/select/parser.h index 61cc6e05876..b25ec951968 100644 --- a/document/src/vespa/document/select/parser.h +++ b/document/src/vespa/document/select/parser.h @@ -2,10 +2,8 @@ #pragma once -#include <memory> -#include <string> -#include <vespa/document/bucket/bucketidfactory.h> #include "node.h" +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/sync.h> diff --git a/document/src/vespa/document/select/result.h b/document/src/vespa/document/select/result.h index ae7b0fdcabf..fbaa8e35eb6 100644 --- a/document/src/vespa/document/select/result.h +++ b/document/src/vespa/document/select/result.h @@ -19,11 +19,9 @@ #pragma once -#include <string> #include <vespa/document/util/printable.h> -namespace document { -namespace select { +namespace document::select { class Result : public Printable { public: @@ -41,16 +39,9 @@ public: const Result& operator!() const; static const Result& get(bool b) { return (b ? True : False); } + static constexpr uint32_t enumRange = 3u; - static uint32_t - enumRange(void) - { - return 3u; - } - - uint32_t - toEnum(void) const - { + uint32_t toEnum() const { if (this == &Result::Invalid) return 0u; if (this == &Result::False) @@ -60,9 +51,7 @@ public: abort(); } - static const Result & - fromEnum(uint32_t val) - { + static const Result &fromEnum(uint32_t val) { if (val == 0u) return Result::Invalid; if (val == 1u) @@ -74,12 +63,9 @@ public: private: Result(); - - // Singletons are not copyable - Result(const Result&); - Result& operator=(const Result&); + // Singletons are not copyable + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; }; -} // select -} // document - +} diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp index f8224b7a158..2cde06fa2f2 100644 --- a/document/src/vespa/document/select/resultlist.cpp +++ b/document/src/vespa/document/select/resultlist.cpp @@ -15,8 +15,7 @@ ResultList::ResultList(const Result& result) { } void -ResultList::add(const FieldValue::IteratorHandler::VariableMap& variables, - const Result& result) +ResultList::add(const fieldvalue::VariableMap& variables, const Result& result) { _results.push_back(ResultPair(variables, &result)); } @@ -27,8 +26,7 @@ ResultList::print(std::ostream& out, bool, const std::string&) const out << "ResultList("; for (uint32_t i = 0; i < _results.size(); i++) { if (!_results[i].first.empty()) { - out << FieldValue::IteratorHandler::toString(_results[i].first) - << " => "; + out << _results[i].first.toString() << " => "; } out << _results[i].second->toString() << " "; } @@ -66,15 +64,12 @@ ResultList::combineResults() const { bool ResultList::combineVariables( - FieldValue::IteratorHandler::VariableMap& output, - const FieldValue::IteratorHandler::VariableMap& input) const + fieldvalue::VariableMap& output, + const fieldvalue::VariableMap& input) const { // First, verify that all variables are overlapping - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = output.begin(); iter != output.end(); iter++) - { - FieldValue::IteratorHandler::VariableMap::const_iterator found( - input.find(iter->first)); + for (fieldvalue::VariableMap::const_iterator iter = output.begin(); iter != output.end(); iter++) { + fieldvalue::VariableMap::const_iterator found(input.find(iter->first)); if (found != input.end()) { if (!(found->second == iter->second)) { @@ -83,11 +78,8 @@ ResultList::combineVariables( } } - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = input.begin(); iter != input.end(); iter++) - { - FieldValue::IteratorHandler::VariableMap::const_iterator found( - output.find(iter->first)); + for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) { + fieldvalue::VariableMap::const_iterator found(output.find(iter->first)); if (found != output.end()) { if (!(found->second == iter->second)) { return false; @@ -95,9 +87,7 @@ ResultList::combineVariables( } } // Ok, variables are overlapping. Add all variables from input to output. - for (FieldValue::IteratorHandler::VariableMap::const_iterator iter - = input.begin(); iter != input.end(); iter++) - { + for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) { output[iter->first] = iter->second; } @@ -112,7 +102,7 @@ ResultList::operator&&(const ResultList& other) const // TODO: optimize for (const auto & it : _results) { for (const auto & it2 : other._results) { - FieldValue::IteratorHandler::VariableMap vars = it.first; + fieldvalue::VariableMap vars = it.first; if (combineVariables(vars, it2.first)) { result.add(vars, *it.second && *it2.second); @@ -131,7 +121,7 @@ ResultList::operator||(const ResultList& other) const // TODO: optimize for (const auto & it : _results) { for (const auto & it2 : other._results) { - FieldValue::IteratorHandler::VariableMap vars = it.first; + fieldvalue::VariableMap vars = it.first; if (combineVariables(vars, it2.first)) { result.add(vars, *it.second || *it2.second); diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h index 6a3cc1f78d1..a00419d9196 100644 --- a/document/src/vespa/document/select/resultlist.h +++ b/document/src/vespa/document/select/resultlist.h @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> -#include <vespa/document/fieldvalue/fieldvalue.h> #include "result.h" +#include <vector> +#include <vespa/document/fieldvalue/variablemap.h> -namespace document { -namespace select { +namespace document::select { class ResultList : public Printable { public: - typedef FieldValue::IteratorHandler::VariableMap VariableMap; + using VariableMap = fieldvalue::VariableMap; typedef std::pair<VariableMap, const Result*> ResultPair; typedef std::vector<ResultPair> Results; typedef Results::iterator iterator; @@ -49,7 +48,7 @@ public: private: Results _results; - bool combineVariables(VariableMap& output, const FieldValue::IteratorHandler::VariableMap& input) const; + bool combineVariables(VariableMap& output, const VariableMap& input) const; }; inline bool operator==(const ResultList& list, const Result& other) { @@ -61,5 +60,4 @@ inline bool operator!=(const ResultList& list, const Result& other) { } } -} diff --git a/document/src/vespa/document/select/resultset.cpp b/document/src/vespa/document/select/resultset.cpp index f7fd3e95fbd..ddde4b366a3 100644 --- a/document/src/vespa/document/select/resultset.cpp +++ b/document/src/vespa/document/select/resultset.cpp @@ -2,9 +2,7 @@ #include "resultset.h" -namespace document { - -namespace select { +namespace document::select { std::vector<ResultSet> ResultSet::_ands; std::vector<ResultSet> ResultSet::_ors; @@ -15,9 +13,9 @@ std::vector<ResultSet> ResultSet::_nots; * inputs. */ void -ResultSet::preCalc(void) +ResultSet::preCalc() { - uint32_t erange = Result::enumRange(); + uint32_t erange = Result::enumRange; uint32_t range = illegalMask(); _ands.resize(range * range); _ors.resize(range * range); @@ -53,13 +51,12 @@ ResultSet::preCalc(void) } -namespace -{ +namespace { class Precalc { public: - Precalc(void) + Precalc() { ResultSet::preCalc(); } @@ -70,5 +67,4 @@ Precalc precalc; } -} // select -} // document +} diff --git a/document/src/vespa/document/select/resultset.h b/document/src/vespa/document/select/resultset.h index b3e4d8c4066..125582faba3 100644 --- a/document/src/vespa/document/select/resultset.h +++ b/document/src/vespa/document/select/resultset.h @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string> -#include <vector> #include "result.h" +#include <vector> -namespace document { -namespace select { +namespace document::select { /* * Contains possible values of operations returning a Result. @@ -29,7 +27,7 @@ public: } static uint32_t illegalMask() { - return (1u << Result::enumRange()); + return (1u << Result::enumRange); } void add(const Result &rhs) { @@ -54,26 +52,24 @@ public: // calculcate set of results emitted by document selection and operator. ResultSet calcAnd(const ResultSet &rhs) const { - return _ands[(_val << Result::enumRange()) | rhs._val]; + return _ands[(_val << Result::enumRange) | rhs._val]; } // calculcate set of results emitted by document selection or operator. ResultSet calcOr(const ResultSet &rhs) const { - return _ors[(_val << Result::enumRange()) | rhs._val]; + return _ors[(_val << Result::enumRange) | rhs._val]; } // calculcate set of results emitted by document selection not operator. - ResultSet calcNot(void) const { return _nots[_val]; } + ResultSet calcNot() const { return _nots[_val]; } - void clear(void) { _val = 0; } - void fill(void) { _val = illegalMask() - 1; } - static void preCalc(void); + void clear() { _val = 0; } + void fill() { _val = illegalMask() - 1; } + static void preCalc(); private: // precalc helper methods void pcnext(void) { ++_val; } bool pcvalid(void) const { return _val < illegalMask(); } }; -} // select -} // document - +} diff --git a/document/src/vespa/document/select/simpleparser.h b/document/src/vespa/document/select/simpleparser.h index e6457c3dec7..c52aed0f367 100644 --- a/document/src/vespa/document/select/simpleparser.h +++ b/document/src/vespa/document/select/simpleparser.h @@ -2,17 +2,11 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> #include "node.h" -#include "valuenode.h" +#include "valuenodes.h" #include "operator.h" -#include <vespa/document/bucket/bucketidfactory.h> -namespace document { - -namespace select { - -namespace simple { +namespace document::select::simple { class Parser { public: @@ -89,9 +83,4 @@ private: const BucketIdFactory & _bucketIdFactory; }; - - -} // simple -} // select -} // parser - +} diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp index 7f85d8b7ebb..2f2468e5afc 100644 --- a/document/src/vespa/document/select/traversingvisitor.cpp +++ b/document/src/vespa/document/select/traversingvisitor.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "traversingvisitor.h" -#include "valuenode.h" +#include "valuenodes.h" #include "branch.h" #include "compare.h" -namespace document { - -namespace select { +namespace document::select { void TraversingVisitor::visitAndBranch(const And &expr) @@ -127,5 +125,3 @@ TraversingVisitor::visitInvalidValueNode(const InvalidValueNode &) } } - -} diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index 7638ba3c36e..3540935f8c7 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -2,7 +2,7 @@ #include "value.h" #include "operator.h" -#include <cstdint> +#include <vespa/document/fieldvalue/fieldvalue.h> namespace document { namespace select { diff --git a/document/src/vespa/document/select/value.h b/document/src/vespa/document/select/value.h index 15856648985..b26e959ea1d 100644 --- a/document/src/vespa/document/select/value.h +++ b/document/src/vespa/document/select/value.h @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + /** * @class document::select::Value * @ingroup select @@ -190,7 +191,7 @@ inline ResultList FloatValue::operator==(const FloatValue& value) const class ArrayValue : public Value { public: - typedef std::pair<FieldValue::IteratorHandler::VariableMap, Value::SP> VariableValue; + using VariableValue = std::pair<fieldvalue::VariableMap, Value::SP>; ArrayValue(const std::vector<VariableValue>& values); diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp index ebf0a466ad9..79b795ca76f 100644 --- a/document/src/vespa/document/select/valuenode.cpp +++ b/document/src/vespa/document/select/valuenode.cpp @@ -1,1168 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "valuenode.h" -#include "visitor.h" -#include "parser.h" -#include <vespa/document/base/exceptions.h> -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/vespalib/util/md5.h> -#include <vespa/document/util/stringutil.h> -#include <vespa/vespalib/text/lowercase.h> -#include <regex> -#include <iomanip> - - -#include <vespa/log/log.h> -LOG_SETUP(".document.select.valuenode"); - -namespace document { -namespace select { - -namespace { - static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*"); -} - -namespace { - bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name) - { - if (type.getName() == name) return true; - for (std::vector<const DocumentType *>::const_iterator it - = type.getInheritedTypes().begin(); - it != type.getInheritedTypes().end(); ++it) - { - if (documentTypeEqualsName(**it, name)) return true; - } - return false; - } -} - -InvalidValueNode::InvalidValueNode(const vespalib::stringref & name) - : _name(name) -{ } - - -void -InvalidValueNode::visit(Visitor &visitor) const -{ - visitor.visitInvalidValueNode(*this); -} - - -void -InvalidValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _name; - if (hadParentheses()) out << ')'; -} - -NullValueNode::NullValueNode(const vespalib::stringref & name) - : _name(name) -{ } - - -void -NullValueNode::visit(Visitor &visitor) const -{ - visitor.visitNullValueNode(*this); -} - - -void -NullValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _name; - if (hadParentheses()) out << ')'; -} - -StringValueNode::StringValueNode(const vespalib::stringref & val) - : _value(val) -{ -} - - -void -StringValueNode::visit(Visitor &visitor) const -{ - visitor.visitStringValueNode(*this); -} - - -void -StringValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << "\"" << StringUtil::escape(_value) << "\""; - if (hadParentheses()) out << ')'; -} - - -void -IntegerValueNode::visit(Visitor &visitor) const -{ - visitor.visitIntegerValueNode(*this); -} - - -void -IntegerValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _value; - if (hadParentheses()) out << ')'; -} - -int64_t -CurrentTimeValueNode::getValue() const -{ - struct timeval mytime; - gettimeofday(&mytime, 0); - return mytime.tv_sec; -} - - -void -CurrentTimeValueNode::visit(Visitor &visitor) const -{ - visitor.visitCurrentTimeValueNode(*this); -} - - -void -CurrentTimeValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "now()"; -} - -std::unique_ptr<Value> -VariableValueNode::getValue(const Context& context) const { - VariableMap::const_iterator iter = context._variables.find(_value); - - if (iter != context._variables.end()) { - return std::unique_ptr<Value>(new FloatValue(iter->second)); - } else { - return std::unique_ptr<Value>(new FloatValue(0.0)); - } -} - - -void -VariableValueNode::visit(Visitor &visitor) const -{ - visitor.visitVariableValueNode(*this); -} - - -void -VariableValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << "$" << _value; - if (hadParentheses()) out << ')'; -} - - -void -FloatValueNode::visit(Visitor &visitor) const -{ - visitor.visitFloatValueNode(*this); -} - - -void -FloatValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _value; - if (hadParentheses()) out << ')'; -} - -FieldValueNode::FieldValueNode(const vespalib::string& doctype, - const vespalib::string& fieldExpression) - : _doctype(doctype), - _fieldExpression(fieldExpression), - _fieldName(extractFieldName(fieldExpression)) -{ -} - -FieldValueNode::FieldValueNode(const FieldValueNode &) = default; -FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default; - -FieldValueNode::~FieldValueNode() {} -vespalib::string -FieldValueNode::extractFieldName(const std::string & fieldExpression) { - std::smatch match; - - if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) { - return vespalib::string(match[1].first, match[1].second); - } - - throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'"); -} - -void -FieldValueNode::initFieldPath(const DocumentType& type) const { - if (_fieldPath.size() == 0) { - FieldPath::UP path(type.buildFieldPath(_fieldExpression)); - if (!path.get()) { - throw FieldNotFoundException( - vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'", - type.toString().c_str(), _fieldExpression.c_str()), - VESPA_STRLOC); - } - _fieldPath = *path; - } -} - -std::unique_ptr<Value> -FieldValueNode::getValue(const Context& context) const -{ - if (context._doc == NULL) { - return std::unique_ptr<Value>(new InvalidValue()); - } - - const Document& doc = *context._doc; - - if (!documentTypeEqualsName(doc.getType(), _doctype)) { - return std::unique_ptr<Value>(new InvalidValue()); - } - try{ - initFieldPath(doc.getType()); - - IteratorHandler handler; - doc.iterateNested(_fieldPath.getFullRange(), handler); - - if (handler.hasSingleValue()) { - return handler.getSingleValue(); - } else { - const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - - if (values.size() == 0) { - return std::unique_ptr<Value>(new NullValue()); - } else { - return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); - } - } - } catch (vespalib::IllegalArgumentException& e) { - LOG(warning, "Caught exception while fetching field from document: %s", e.what()); - return std::unique_ptr<Value>(new InvalidValue()); - } catch (FieldNotFoundException& e) { - LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str()); - return std::unique_ptr<Value>(new InvalidValue()); - } -} - -FieldValueNode::IteratorHandler::IteratorHandler() { } -FieldValueNode::IteratorHandler::~IteratorHandler() { } - -bool -FieldValueNode::IteratorHandler::hasSingleValue() const { - return _firstValue.get() && (_values.size() == 0); -} - -std::unique_ptr<Value> -FieldValueNode::IteratorHandler::getSingleValue() { - return std::move(_firstValue); -} - -const std::vector<ArrayValue::VariableValue>& -FieldValueNode::IteratorHandler::getValues() { - if (_firstValue.get()) { - _values.insert(_values.begin(), ArrayValue::VariableValue(FieldValue::IteratorHandler::VariableMap(), Value::SP(_firstValue.release()))); - } - - return _values; -} - -void -FieldValueNode::IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { - (void) fid; - if (!_firstValue && getVariables().empty()) { - _firstValue = getInternalValue(fv.getValue()); - } else { - _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); - } -} - -std::unique_ptr<Value> -FieldValueNode::IteratorHandler::getInternalValue(const FieldValue& fval) const -{ - switch(fval.getClass().id()) { - case document::IntFieldValue::classId: - { - const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsInt(), false); - } - case document::ByteFieldValue::classId: - { - const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsByte(), false); - } - case LongFieldValue::classId: - { - const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval)); - return std::make_unique<IntegerValue>(val.getAsLong(), false); - } - case FloatFieldValue::classId: - { - const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval)); - return std::make_unique<FloatValue>(val.getAsFloat()); - } - case DoubleFieldValue::classId: - { - const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval)); - return std::make_unique<FloatValue>(val.getAsDouble()); - } - case StringFieldValue::classId: - { - const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval)); - return std::make_unique<StringValue>(val.getAsString()); - } - case ArrayFieldValue::classId: - { - const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval)); - if (val.size() == 0) { - return std::make_unique<NullValue>(); - } else { - std::vector<ArrayValue::VariableValue> values; - // TODO: Array comparison. - return std::make_unique<ArrayValue>(values); - } - } - case StructFieldValue::classId: - { - const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval)); - if (val.empty()) { - return std::make_unique<NullValue>(); - } else { - StructValue::ValueMap values; - for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) { - FieldValue::UP fv(val.getValue(it.field())); - values[it.field().getName()] = Value::SP(getInternalValue(*fv).release()); - } - return std::make_unique<StructValue>(values); - } - } - case MapFieldValue::classId: - { - const MapFieldValue& val(static_cast<const MapFieldValue&>(fval)); - if (val.isEmpty()) { - return std::make_unique<NullValue>(); - } else { - std::vector<ArrayValue::VariableValue> values; - // TODO: Map comparison - return std::make_unique<ArrayValue>(values); - } - } - } - LOG(warning, "Tried to use unsupported datatype %s in field comparison", - fval.getDataType()->toString().c_str()); - return std::make_unique<InvalidValue>(); -} - - -void -FieldValueNode::visit(Visitor &visitor) const -{ - visitor.visitFieldValueNode(*this); -} - - -void -FieldValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _doctype << "." << _fieldExpression; - if (hadParentheses()) out << ')'; -} - - -std::unique_ptr<Value> -FieldValueNode::traceValue(const Context &context, std::ostream& out) const -{ - if (context._doc == NULL) { - return defaultTrace(getValue(context), out); - } - const Document &doc(*context._doc); - if (!documentTypeEqualsName(doc.getType(), _doctype)) { - out << "Document is of type " << doc.getType() << " which isn't a " - << _doctype << " document, thus resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue()); - } - try{ - initFieldPath(doc.getType()); - - IteratorHandler handler; - doc.iterateNested(_fieldPath.getFullRange(), handler); - - if (handler.hasSingleValue()) { - return handler.getSingleValue(); - } else { - const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - - if (values.size() == 0) { - return std::unique_ptr<Value>(new NullValue()); - } else { - return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); - } - } - } catch (FieldNotFoundException& e) { - LOG(warning, "Tried to compare to field %s, not found in document type", - _fieldExpression.c_str()); - out << "Field not found in document type " << doc.getType() - << ". Returning invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue()); - } -} - -IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, const vespalib::stringref & type, - int widthBits, int divisionBits) - : _bucketIdFactory(bucketIdFactory), - _id(name), - _typestring(type), - _type(ALL), - _widthBits(widthBits), - _divisionBits(divisionBits) -{ - if (type.length() > 2) switch (type[0]) { - case 'b': _type = BUCKET; - break; - case 'n': _type = NS; - break; - case 'g': - if (type[1] == 'r') { - _type = GROUP; - } else if (type[1] == 'i') { - _type = GID; - } - break; - case 's': { - if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; } - break; - } - case 't': - _type = TYPE; - break; - case 'u': - _type = USER; - break; - case 'o': - _type = ORDER; - break; - } -} - - -std::unique_ptr<Value> -IdValueNode::getValue(const Context& context) const -{ - if (context._doc != NULL) { - return getValue(context._doc->getId()); - } else if (context._docId != NULL) { - return getValue(*context._docId); - } else { - return getValue(context._docUpdate->getId()); - } -} - - -std::unique_ptr<Value> -IdValueNode::getValue(const DocumentId& id) const -{ - vespalib::string value; - switch (_type) { - case BUCKET: - return std::unique_ptr<Value>( - new IntegerValue( - _bucketIdFactory.getBucketId(id).getId(), true)); - case NS: - value = id.getScheme().getNamespace(); break; - case SCHEME: - value = id.getScheme().getTypeName(id.getScheme().getType()); - break; - case TYPE: - if (id.getScheme().hasDocType()) { - value = id.getScheme().getDocType(); - } else { - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case SPEC: - value = id.getScheme().getNamespaceSpecific(); - break; - case ALL: - value = id.getScheme().toString(); - break; - case GROUP: - if (id.getScheme().hasGroup()) { - value = id.getScheme().getGroup(); - } else { - fprintf(stderr, "***** Returning invalid value for %s\n", - id.toString().c_str()); - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case GID: - value = id.getGlobalId().toString(); - break; - case ORDER: - if (id.getScheme().getType() == IdString::ORDERDOC) { - const OrderDocIdString& ods( - static_cast<const OrderDocIdString&>(id.getScheme())); - if (ods.getWidthBits() == _widthBits - && ods.getDivisionBits() == _divisionBits) - { - return std::unique_ptr<Value>(new IntegerValue( - static_cast<const OrderDocIdString&>(id.getScheme()) - .getOrdering(), false)); - } - } - return std::unique_ptr<Value>(new InvalidValue()); - case USER: - if (id.getScheme().hasNumber()) { - return std::unique_ptr<Value>( - new IntegerValue(id.getScheme().getNumber(), false)); - } else { - return std::unique_ptr<Value>(new InvalidValue); - } - } - - return std::unique_ptr<Value>(new StringValue(value)); -} - - -std::unique_ptr<Value> -IdValueNode::traceValue(const Context& context, - std::ostream &out) const -{ - if (context._doc != NULL) { - return traceValue(context._doc->getId(), out); - } else if (context._docId != NULL) { - return traceValue(*context._docId, out); - } else { - return traceValue(context._docUpdate->getId(), out); - } -} - - -std::unique_ptr<Value> -IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const -{ - vespalib::string value; - switch (_type) { - case BUCKET: - { - document::BucketId bucket(_bucketIdFactory.getBucketId(id)); - std::unique_ptr<Value> result( - new IntegerValue(bucket.getId(), true)); - out << "Found id.bucket specification. Resolved to " - << bucket.toString() << ".\n"; - return result; - } - case NS: - value = id.getScheme().getNamespace(); - out << "Resolved id.namespace to value\"" << value << "\".\n"; - break; - case SCHEME: - value = id.getScheme().getTypeName(id.getScheme().getType()); - out << "Resolved id.scheme to value\"" << value << "\".\n"; - break; - case TYPE: - if (id.getScheme().hasDocType()) { - value = id.getScheme().getDocType(); - out << "Resolved id.type to value\"" << value << "\".\n"; - } else { - out << "Could not resolve type of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case SPEC: - value = id.getScheme().getNamespaceSpecific(); - out << "Resolved id.specific to value\"" << value << "\".\n"; - break; - case ALL: - value = id.getScheme().toString(); - out << "Resolved id to \"" << value << "\".\n"; - break; - case GROUP: - if (id.getScheme().hasGroup()) { - value = id.getScheme().getGroup(); - out << "Resolved group of doc (type " << id.getScheme().getType() - << ") to \"" << value << "\".\n"; - } else { - out << "Can't resolve group of doc \"" << id << "\".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - break; - case GID: - value = id.getGlobalId().toString(); - out << "Resolved gid to \"" << value << "\".\n"; - break; - case ORDER: - if (id.getScheme().getType() == IdString::ORDERDOC) { - const OrderDocIdString& ods( - static_cast<const OrderDocIdString&>(id.getScheme())); - if (ods.getWidthBits() == _widthBits - && ods.getDivisionBits() == _divisionBits) - { - std::unique_ptr<Value> result(new IntegerValue( - static_cast<const OrderDocIdString&>(id.getScheme()) - .getOrdering(), false)); - out << "Resolved id.order to int " << *result << "\n"; - return result; - } - } - out << "Could not resolve id.order(" << _widthBits << ", " - << _divisionBits << ") of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue()); - case USER: - if (id.getScheme().hasNumber()) { - std::unique_ptr<Value> result( - new IntegerValue(id.getScheme().getNumber(), false)); - out << "Resolved user of doc type " << id.getScheme().getType() - << " to " << *result << ".\n"; - return result; - } else { - out << "Could not resolve user of doc " << id << ".\n"; - return std::unique_ptr<Value>(new InvalidValue); - } - } - - return std::unique_ptr<Value>(new StringValue(value)); -} - - -void -IdValueNode::visit(Visitor &visitor) const -{ - visitor.visitIdValueNode(*this); -} - - -void -IdValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _id; - if (_type != ALL) { - out << '.' << _typestring; - } - if (_type == ORDER) { - out << "(" << _widthBits << "," << _divisionBits << ")"; - } - if (hadParentheses()) out << ')'; -} - -SearchColumnValueNode::SearchColumnValueNode( - const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, int numColumns) - : _bucketIdFactory(bucketIdFactory), - _id(name), - _numColumns(numColumns), - _distribution(_numColumns, 16) -{ -} - -int64_t -SearchColumnValueNode::getValue(const BucketId& id) const -{ - return _distribution.getColumn(id); -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const Context& context) const -{ - if (context._doc != NULL) { - return getValue(context._doc->getId()); - } else if (context._docId != NULL) { - return getValue(*context._docId); - } else { - return getValue(context._docUpdate->getId()); - } -} - - -std::unique_ptr<Value> -SearchColumnValueNode::getValue(const DocumentId& id) const -{ - return std::unique_ptr<Value>(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); -} +#include <ostream> +namespace document::select { std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const Context& context, - std::ostream &out) const -{ - if (context._doc != NULL) { - return traceValue(context._doc->getId(), out); - } else if (context._docId != NULL) { - return traceValue(*context._docId, out); - } else { - return traceValue(context._docUpdate->getId(), out); - } +ValueNode::traceValue(const Context &context, std::ostream &out) const { + return defaultTrace(getValue(context), out); } - std::unique_ptr<Value> -SearchColumnValueNode::traceValue(const DocumentId& id, - std::ostream& out) const +ValueNode::defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const { - std::unique_ptr<Value> result(new IntegerValue( - getValue(_bucketIdFactory.getBucketId(id)), false)); - out << "Resolved search column of doc \"" << id << "\" to " << *result - << "\n"; - return result; -} - - -void -SearchColumnValueNode::visit(Visitor &visitor) const -{ - visitor.visitSearchColumnValueNode(*this); -} - - -void -SearchColumnValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - if (hadParentheses()) out << '('; - out << _id; - out << '.' << _numColumns; - if (hadParentheses()) out << ')'; -} - -namespace { - union HashUnion { - unsigned char _key[16]; - int64_t _hash[2]; - }; - int64_t hash(const void* data, uint32_t len) { - HashUnion hash; - fastc_md5sum((const unsigned char*) data, len, hash._key); - return hash._hash[0]; - } + out << "Returning value " << *val << ".\n"; + return std::move(val); } -FunctionValueNode::FunctionValueNode(const vespalib::stringref & name, - std::unique_ptr<ValueNode> src) - : _function(), - _funcname(name), - _source(std::move(src)) -{ - if (name == "lowercase") { - _function = LOWERCASE; - } else if (name == "hash") { - _function = HASH; - } else if (name == "abs") { - _function = ABS; - } else { - throw ParsingFailedException("No function '"+name+"' exist.", - VESPA_STRLOC); - } -} - -std::unique_ptr<Value> -FunctionValueNode::getValue(std::unique_ptr<Value> val) const -{ - switch (val->getType()) { - case Value::String: - { - StringValue& sval(static_cast<StringValue&>(*val)); - if (_function == LOWERCASE) { - return std::unique_ptr<Value>(new StringValue( - vespalib::LowerCase::convert(sval.getValue()))); - } else if (_function == HASH) { - return std::unique_ptr<Value>(new IntegerValue( - hash(sval.getValue().c_str(), sval.getValue().size()), - false)); - } - break; - } - case Value::Float: - { - FloatValue& fval(static_cast<FloatValue&>(*val)); - if (_function == HASH) { - FloatValue::ValueType ffval = fval.getValue(); - return std::unique_ptr<Value>(new IntegerValue( - hash(&ffval, sizeof(ffval)), false)); - } else if (_function == ABS) { - FloatValue::ValueType ffval = fval.getValue(); - if (ffval < 0) ffval *= -1; - return std::unique_ptr<Value>(new FloatValue(ffval)); - } - break; - } - case Value::Integer: - { - IntegerValue& ival(static_cast<IntegerValue&>(*val)); - if (_function == HASH) { - IntegerValue::ValueType iival = ival.getValue(); - return std::unique_ptr<Value>(new IntegerValue( - hash(&iival, sizeof(iival)), false)); - } else if (_function == ABS) { - IntegerValue::ValueType iival = ival.getValue(); - if (iival < 0) iival *= -1; - return std::unique_ptr<Value>(new IntegerValue(iival, false)); - } - break; - } - case Value::Bucket: - { - throw ParsingFailedException( - "No functioncalls are allowed on value of type bucket", - VESPA_STRLOC); - break; - } - - case Value::Array: break; - case Value::Struct: break; - case Value::Invalid: break; - case Value::Null: break; - } - return std::unique_ptr<Value>(new InvalidValue); -} - -std::unique_ptr<Value> -FunctionValueNode::traceValue(std::unique_ptr<Value> val, - std::ostream& out) const -{ - switch (val->getType()) { - case Value::String: - { - StringValue& sval(static_cast<StringValue&>(*val)); - if (_function == LOWERCASE) { - std::unique_ptr<Value> result(new StringValue( - vespalib::LowerCase::convert(sval.getValue()))); - out << "Performed lowercase function on '" << sval - << "' => '" << *result << "'.\n"; - return result; - } else if (_function == HASH) { - std::unique_ptr<Value> result(new IntegerValue( - hash(sval.getValue().c_str(), sval.getValue().size()), - false)); - out << "Performed hash on string '" << sval << "' -> " - << *result << "\n"; - return result; - } - break; - } - case Value::Float: - { - FloatValue& fval(static_cast<FloatValue&>(*val)); - if (_function == HASH) { - FloatValue::ValueType ffval = fval.getValue(); - std::unique_ptr<Value> result(new IntegerValue( - hash(&ffval, sizeof(ffval)), false)); - out << "Performed hash on float " << ffval << " -> " << *result - << "\n"; - return result; - } else if (_function == ABS) { - FloatValue::ValueType ffval = fval.getValue(); - if (ffval < 0) ffval *= -1; - out << "Performed abs on float " << fval.getValue() << " -> " - << ffval << "\n"; - return std::unique_ptr<Value>(new FloatValue(ffval)); - } - break; - } - case Value::Integer: - { - IntegerValue& ival(static_cast<IntegerValue&>(*val)); - if (_function == HASH) { - IntegerValue::ValueType iival = ival.getValue(); - std::unique_ptr<Value> result(new IntegerValue( - hash(&iival, sizeof(iival)), false)); - out << "Performed hash on float " << iival << " -> " << *result - << "\n"; - return result; - } else if (_function == ABS) { - IntegerValue::ValueType iival = ival.getValue(); - if (iival < 0) iival *= -1; - out << "Performed abs on integer " << ival.getValue() << " -> " - << iival << "\n"; - return std::unique_ptr<Value>(new IntegerValue(iival, false)); - } - break; - } - case Value::Bucket: break; - case Value::Array: break; - case Value::Struct: break; - case Value::Invalid: break; - case Value::Null: break; - } - out << "Cannot use function " << _function << " on a value of type " - << val->getType() << ". Resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue); -} - - -void -FunctionValueNode::visit(Visitor &visitor) const -{ - visitor.visitFunctionValueNode(*this); -} - - -void -FunctionValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - if (hadParentheses()) out << '('; - _source->print(out, verbose, indent); - out << '.' << _funcname << "()"; - if (hadParentheses()) out << ')'; -} - -ArithmeticValueNode::ArithmeticValueNode( - std::unique_ptr<ValueNode> left, const vespalib::stringref & op, - std::unique_ptr<ValueNode> right) - : _operator(), - _left(std::move(left)), - _right(std::move(right)) -{ - if (op.size() == 1) switch (op[0]) { - case '+': _operator = ADD; return; - case '-': _operator = SUB; return; - case '*': _operator = MUL; return; - case '/': _operator = DIV; return; - case '%': _operator = MOD; return; - } - throw ParsingFailedException( - "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC); -} - -const char* -ArithmeticValueNode::getOperatorName() const -{ - switch (_operator) { - case ADD: return "+"; - case SUB: return "-"; - case MUL: return "*"; - case DIV: return "/"; - case MOD: return "%"; - } - return "UNKNOWN"; -} - - - -std::unique_ptr<Value> -ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval) const -{ - switch (_operator) { - case ADD: - { - if (lval->getType() == Value::String && - rval->getType() == Value::String) - { - StringValue& slval(static_cast<StringValue&>(*lval)); - StringValue& srval(static_cast<StringValue&>(*rval)); - return std::unique_ptr<Value>(new StringValue( - slval.getValue() + srval.getValue())); - } - } - case SUB: - case MUL: - case DIV: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - IntegerValue::ValueType res = 0; - switch (_operator) { - case ADD: res = ilval.getValue() + irval.getValue(); break; - case SUB: res = ilval.getValue() - irval.getValue(); break; - case MUL: res = ilval.getValue() * irval.getValue(); break; - case DIV: - if (irval.getValue() != 0) { - res = ilval.getValue() / irval.getValue(); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - break; - case MOD: assert(0); - } - return std::unique_ptr<Value>(new IntegerValue(res, false)); - } - NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); - NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get())); - if (nlval != 0 && nrval != 0) { - NumberValue::CommonValueType res = 0; - switch (_operator) { - case ADD: res = nlval->getCommonValue() - + nrval->getCommonValue(); break; - case SUB: res = nlval->getCommonValue() - - nrval->getCommonValue(); break; - case MUL: res = nlval->getCommonValue() - * nrval->getCommonValue(); break; - case DIV: - if (nrval->getCommonValue() != 0) { - res = nlval->getCommonValue() - / nrval->getCommonValue(); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - break; - case MOD: assert(0); - } - return std::unique_ptr<Value>(new FloatValue(res)); - } - } - case MOD: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - if (irval.getValue() != 0) { - return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false)); - } else { - throw vespalib::IllegalArgumentException("Division by zero"); - } - } - } - } - return std::unique_ptr<Value>(new InvalidValue); -} - -std::unique_ptr<Value> -ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval, - std::ostream& out) const -{ - switch (_operator) { - case ADD: - { - if (lval->getType() == Value::String && - rval->getType() == Value::String) - { - StringValue& slval(static_cast<StringValue&>(*lval)); - StringValue& srval(static_cast<StringValue&>(*rval)); - std::unique_ptr<Value> result(new StringValue( - slval.getValue() + srval.getValue())); - out << "Appended strings '" << slval << "' + '" << srval - << "' -> '" << *result << "'.\n"; - return result; - } - } - case SUB: - case MUL: - case DIV: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - IntegerValue::ValueType res = 0; - switch (_operator) { - case ADD: res = ilval.getValue() + irval.getValue(); break; - case SUB: res = ilval.getValue() - irval.getValue(); break; - case MUL: res = ilval.getValue() * irval.getValue(); break; - case DIV: res = ilval.getValue() / irval.getValue(); break; - case MOD: assert(0); - } - std::unique_ptr<Value> result(new IntegerValue(res, false)); - out << "Performed integer operation " << ilval << " " - << getOperatorName() << " " << irval << " = " << *result - << "\n"; - return result; - } - NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); - NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get())); - if (nlval != 0 && nrval != 0) { - NumberValue::CommonValueType res = 0; - switch (_operator) { - case ADD: res = nlval->getCommonValue() - + nrval->getCommonValue(); break; - case SUB: res = nlval->getCommonValue() - - nrval->getCommonValue(); break; - case MUL: res = nlval->getCommonValue() - * nrval->getCommonValue(); break; - case DIV: res = nlval->getCommonValue() - / nrval->getCommonValue(); break; - case MOD: assert(0); - } - std::unique_ptr<Value> result(new FloatValue(res)); - out << "Performed float operation " << nlval << " " - << getOperatorName() << " " << nrval << " = " << *result - << "\n"; - return result; - } - } - case MOD: - { - if (lval->getType() == Value::Integer && - rval->getType() == Value::Integer) - { - IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); - IntegerValue& irval(static_cast<IntegerValue&>(*rval)); - std::unique_ptr<Value> result(new IntegerValue( - ilval.getValue() % irval.getValue(), false)); - out << "Performed integer operation " << ilval << " " - << getOperatorName() << " " << irval << " = " << *result - << "\n"; - return result; - } - } - } - out << "Failed to do operation " << getOperatorName() - << " on values of type " << lval->getType() << " and " - << rval->getType() << ". Resolving invalid.\n"; - return std::unique_ptr<Value>(new InvalidValue); -} - - -void -ArithmeticValueNode::visit(Visitor &visitor) const -{ - visitor.visitArithmeticValueNode(*this); -} - - -void -ArithmeticValueNode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - if (hadParentheses()) out << '('; - _left->print(out, verbose, indent); - switch (_operator) { - case ADD: out << " + "; break; - case SUB: out << " - "; break; - case MUL: out << " * "; break; - case DIV: out << " / "; break; - case MOD: out << " % "; break; - } - _right->print(out, verbose, indent); - if (hadParentheses()) out << ')'; } -} // select -} // document diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h index e3f0b4c4a94..f36085f769d 100644 --- a/document/src/vespa/document/select/valuenode.h +++ b/document/src/vespa/document/select/valuenode.h @@ -13,51 +13,28 @@ #pragma once #include "value.h" -#include "context.h" -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/bucket/bucketdistribution.h> -#include <vespa/document/bucket/bucketidfactory.h> -namespace document { +namespace document::select { -class Document; -class Field; - -namespace select { +class Context; +class Visitor; class ValueNode : public Printable { public: - typedef std::unique_ptr<ValueNode> UP; - typedef Context::VariableMap VariableMap; + using UP = std::unique_ptr<ValueNode>; ValueNode() : _parentheses(false) {} virtual ~ValueNode() {} void setParentheses() { _parentheses = true; } - - void clearParentheses() - { - _parentheses = false; - } - + void clearParentheses() { _parentheses = false; } bool hadParentheses() const { return _parentheses; } - virtual std::unique_ptr<Value> - getValue(const Context& context) const = 0; - - virtual std::unique_ptr<Value> - traceValue(const Context &context, - std::ostream &out) const { - return defaultTrace(getValue(context), out); - } - - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override = 0; - + virtual std::unique_ptr<Value> getValue(const Context& context) const = 0; virtual void visit(Visitor&) const = 0; - virtual ValueNode::UP clone() const = 0; + virtual std::unique_ptr<Value> traceValue(const Context &context, std::ostream &out) const; private: bool _parentheses; // Set to true if parentheses was used around this part // Set such that we can recreate original query in print. @@ -70,349 +47,8 @@ protected: return ret; } - std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, - std::ostream& out) const - { - out << "Returning value " << *val << ".\n"; - return std::move(val); - } -}; - -class InvalidValueNode : public ValueNode -{ - vespalib::string _name; -public: - InvalidValueNode(const vespalib::stringref & name); - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new InvalidValue()); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new InvalidValueNode(_name)); - } -}; - -class NullValueNode : public ValueNode -{ - vespalib::string _name; -public: - NullValueNode(const vespalib::stringref & name); - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new NullValue()); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new NullValueNode(_name)); - } -}; - -class StringValueNode : public ValueNode -{ - vespalib::string _value; -public: - StringValueNode(const vespalib::stringref & val); - - const vespalib::string& getValue() const { return _value; } - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new StringValue(_value)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new StringValueNode(_value)); - } -}; - -class IntegerValueNode : public ValueNode -{ - int64_t _value; - bool _isBucketValue; -public: - IntegerValueNode(int64_t val, bool isBucketValue) - : _value(val), _isBucketValue(isBucketValue) {} - - int64_t getValue() const { return _value; } - - virtual std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new IntegerValueNode(_value, _isBucketValue)); - } -}; - -class CurrentTimeValueNode : public ValueNode -{ -public: - int64_t getValue() const; - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new IntegerValue(getValue(), false)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new CurrentTimeValueNode); - } -}; - -class VariableValueNode : public ValueNode -{ - vespalib::string _value; -public: - VariableValueNode(const vespalib::string & variableName) : _value(variableName) {} - - const vespalib::string& getVariableName() const { return _value; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new VariableValueNode(_value)); - } -}; - -class FloatValueNode : public ValueNode -{ - double _value; -public: - FloatValueNode(double val) : _value(val) {} - - double getValue() const { return _value; } - - std::unique_ptr<Value> getValue(const Context&) const override { - return std::unique_ptr<Value>(new FloatValue(_value)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FloatValueNode(_value)); - } -}; - -class FieldValueNode : public ValueNode -{ - vespalib::string _doctype; - vespalib::string _fieldExpression; - vespalib::string _fieldName; - mutable FieldPath _fieldPath; - -public: - FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression); - FieldValueNode(const FieldValueNode &); - FieldValueNode & operator = (const FieldValueNode &); - FieldValueNode(FieldValueNode &&) = default; - FieldValueNode & operator = (FieldValueNode &&) = default; - ~FieldValueNode(); - - const vespalib::string& getDocType() const { return _doctype; } - const vespalib::string& getRealFieldName() const { return _fieldName; } - const vespalib::string& getFieldName() const { return _fieldExpression; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); - } - - static vespalib::string extractFieldName(const std::string & fieldExpression); - -private: - class IteratorHandler : public FieldValue::IteratorHandler - { - public: - IteratorHandler(); - ~IteratorHandler(); - bool hasSingleValue() const; - - std::unique_ptr<Value> getSingleValue(); - const std::vector<ArrayValue::VariableValue>& getValues(); - - private: - std::unique_ptr<Value> _firstValue; - std::vector<ArrayValue::VariableValue> _values; - - void onPrimitive(uint32_t fid, const Content & fv) override; - std::unique_ptr<Value> getInternalValue(const FieldValue& fval) const; - }; - - void initFieldPath(const DocumentType&) const; -}; - -class IdValueNode : public ValueNode -{ -public: - enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL }; - - IdValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, const vespalib::stringref & type, - int widthBits = -1, int divisionBits = -1); - - Type getType() const { return _type; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - - std::unique_ptr<Value> getValue(const DocumentId& id) const; - - std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; - - std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); - } - - int getWidthBits() const { return _widthBits; } - int getDivisionBits() const { return _divisionBits; } - -private: - const BucketIdFactory& _bucketIdFactory; - vespalib::string _id; - vespalib::string _typestring; - Type _type; - int _widthBits; - int _divisionBits; + std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const; }; -class SearchColumnValueNode : public ValueNode -{ -public: - SearchColumnValueNode(const BucketIdFactory& bucketIdFactory, - const vespalib::stringref & name, - int numColumns); - - int getColumns() { return _numColumns; } - - std::unique_ptr<Value> getValue(const Context& context) const override; - std::unique_ptr<Value> getValue(const DocumentId& id) const; - std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; - std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; - - int64_t getValue(const BucketId& bucketId) const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns)); } -private: - const BucketIdFactory& _bucketIdFactory; - vespalib::string _id; - int _numColumns; - BucketDistribution _distribution; -}; - -class FunctionValueNode : public ValueNode -{ -public: - enum Function { LOWERCASE, HASH, ABS }; - - FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src); - - Function getFunction() const { return _function; } - const vespalib::string &getFunctionName(void) const { return _funcname; } - - std::unique_ptr<Value> getValue(const Context& context) const override { - return getValue(_source->getValue(context)); - } - - std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override { - return traceValue(_source->getValue(context), out); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new FunctionValueNode(_funcname, _source->clone())); - } - - const ValueNode& getChild() const { return *_source; } - -private: - Function _function; - vespalib::string _funcname; - std::unique_ptr<ValueNode> _source; - - virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const; - virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val, - std::ostream& out) const; -}; - -class ArithmeticValueNode : public ValueNode -{ -public: - enum Operator { ADD, SUB, MUL, DIV, MOD }; - - ArithmeticValueNode(std::unique_ptr<ValueNode> left, - const vespalib::stringref & op, - std::unique_ptr<ValueNode> right); - - Operator getOperator() const { return _operator; } - const char* getOperatorName() const; - - std::unique_ptr<Value> - getValue(const Context& context) const override { - return getValue(_left->getValue(context), _right->getValue(context)); - } - - std::unique_ptr<Value> - traceValue(const Context &context, std::ostream& out) const override { - return traceValue(_left->getValue(context), _right->getValue(context), out); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void visit(Visitor& visitor) const override; - - ValueNode::UP clone() const override { - return wrapParens(new ArithmeticValueNode(_left->clone(), - getOperatorName(), - _right->clone())); - } - - const ValueNode& getLeft() const { return *_left; } - const ValueNode& getRight() const { return *_right; } - -private: - Operator _operator; - std::unique_ptr<ValueNode> _left; - std::unique_ptr<ValueNode> _right; - - virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval) const; - virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval, - std::unique_ptr<Value> rval, - std::ostream&) const; -}; - -} // select -} // document diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp new file mode 100644 index 00000000000..3a4edbef093 --- /dev/null +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -0,0 +1,1183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "valuenodes.h" +#include "visitor.h" +#include "parser.h" +#include <vespa/document/bucket/bucketdistribution.h> +#include <vespa/document/base/exceptions.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/vespalib/util/md5.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/vespalib/text/lowercase.h> +#include <regex> +#include <iomanip> + + +#include <vespa/log/log.h> +LOG_SETUP(".document.select.valuenode"); + +namespace document::select { + +namespace { + static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*"); +} + +namespace { + bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name) + { + if (type.getName() == name) return true; + for (std::vector<const DocumentType *>::const_iterator it + = type.getInheritedTypes().begin(); + it != type.getInheritedTypes().end(); ++it) + { + if (documentTypeEqualsName(**it, name)) return true; + } + return false; + } +} + +InvalidValueNode::InvalidValueNode(const vespalib::stringref & name) + : _name(name) +{ } + + +void +InvalidValueNode::visit(Visitor &visitor) const +{ + visitor.visitInvalidValueNode(*this); +} + + +void +InvalidValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _name; + if (hadParentheses()) out << ')'; +} + +NullValueNode::NullValueNode(const vespalib::stringref & name) + : _name(name) +{ } + + +void +NullValueNode::visit(Visitor &visitor) const +{ + visitor.visitNullValueNode(*this); +} + + +void +NullValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _name; + if (hadParentheses()) out << ')'; +} + +StringValueNode::StringValueNode(const vespalib::stringref & val) + : _value(val) +{ +} + + +void +StringValueNode::visit(Visitor &visitor) const +{ + visitor.visitStringValueNode(*this); +} + + +void +StringValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << "\"" << StringUtil::escape(_value) << "\""; + if (hadParentheses()) out << ')'; +} + + +void +IntegerValueNode::visit(Visitor &visitor) const +{ + visitor.visitIntegerValueNode(*this); +} + + +void +IntegerValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _value; + if (hadParentheses()) out << ')'; +} + +int64_t +CurrentTimeValueNode::getValue() const +{ + struct timeval mytime; + gettimeofday(&mytime, 0); + return mytime.tv_sec; +} + + +void +CurrentTimeValueNode::visit(Visitor &visitor) const +{ + visitor.visitCurrentTimeValueNode(*this); +} + + +void +CurrentTimeValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + out << "now()"; +} + +std::unique_ptr<Value> +VariableValueNode::getValue(const Context& context) const { + return context.getValue(_value); +} + +void +VariableValueNode::visit(Visitor &visitor) const +{ + visitor.visitVariableValueNode(*this); +} + + +void +VariableValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << "$" << _value; + if (hadParentheses()) out << ')'; +} + + +void +FloatValueNode::visit(Visitor &visitor) const +{ + visitor.visitFloatValueNode(*this); +} + + +void +FloatValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _value; + if (hadParentheses()) out << ')'; +} + +FieldValueNode::FieldValueNode(const vespalib::string& doctype, + const vespalib::string& fieldExpression) + : _doctype(doctype), + _fieldExpression(fieldExpression), + _fieldName(extractFieldName(fieldExpression)) +{ +} + +FieldValueNode::FieldValueNode(const FieldValueNode &) = default; +FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default; + +FieldValueNode::~FieldValueNode() {} +vespalib::string +FieldValueNode::extractFieldName(const std::string & fieldExpression) { + std::smatch match; + + if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) { + return vespalib::string(match[1].first, match[1].second); + } + + throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'"); +} + +namespace { + +class IteratorHandler : public fieldvalue::IteratorHandler { +public: + IteratorHandler(); + ~IteratorHandler(); + bool hasSingleValue() const; + std::unique_ptr<Value> getSingleValue(); + const std::vector<ArrayValue::VariableValue> &getValues(); + +private: + std::unique_ptr<Value> _firstValue; + std::vector<ArrayValue::VariableValue> _values; + + void onPrimitive(uint32_t fid, const Content &fv) override; + std::unique_ptr<Value> getInternalValue(const FieldValue &fval) const; +}; + +IteratorHandler::IteratorHandler() { } +IteratorHandler::~IteratorHandler() { } + +bool +IteratorHandler::hasSingleValue() const { + return _firstValue.get() && (_values.size() == 0); +} + +std::unique_ptr<Value> +IteratorHandler::getSingleValue() { + return std::move(_firstValue); +} + +const std::vector<ArrayValue::VariableValue>& +IteratorHandler::getValues() { + if (_firstValue.get()) { + _values.insert(_values.begin(), ArrayValue::VariableValue(fieldvalue::VariableMap(), Value::SP(_firstValue.release()))); + } + + return _values; +} + +void +IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) { + (void) fid; + if (!_firstValue && getVariables().empty()) { + _firstValue = getInternalValue(fv.getValue()); + } else { + _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release())); + } +} + +std::unique_ptr<Value> +IteratorHandler::getInternalValue(const FieldValue& fval) const +{ + switch(fval.getClass().id()) { + case document::IntFieldValue::classId: + { + const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsInt(), false); + } + case document::ByteFieldValue::classId: + { + const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsByte(), false); + } + case LongFieldValue::classId: + { + const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval)); + return std::make_unique<IntegerValue>(val.getAsLong(), false); + } + case FloatFieldValue::classId: + { + const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval)); + return std::make_unique<FloatValue>(val.getAsFloat()); + } + case DoubleFieldValue::classId: + { + const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval)); + return std::make_unique<FloatValue>(val.getAsDouble()); + } + case StringFieldValue::classId: + { + const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval)); + return std::make_unique<StringValue>(val.getAsString()); + } + case ArrayFieldValue::classId: + { + const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval)); + if (val.size() == 0) { + return std::make_unique<NullValue>(); + } else { + std::vector<ArrayValue::VariableValue> values; + // TODO: Array comparison. + return std::make_unique<ArrayValue>(values); + } + } + case StructFieldValue::classId: + { + const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval)); + if (val.empty()) { + return std::make_unique<NullValue>(); + } else { + StructValue::ValueMap values; + for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) { + FieldValue::UP fv(val.getValue(it.field())); + values[it.field().getName()] = Value::SP(getInternalValue(*fv).release()); + } + return std::make_unique<StructValue>(values); + } + } + case MapFieldValue::classId: + { + const MapFieldValue& val(static_cast<const MapFieldValue&>(fval)); + if (val.isEmpty()) { + return std::make_unique<NullValue>(); + } else { + std::vector<ArrayValue::VariableValue> values; + // TODO: Map comparison + return std::make_unique<ArrayValue>(values); + } + } + } + LOG(warning, "Tried to use unsupported datatype %s in field comparison", + fval.getDataType()->toString().c_str()); + return std::make_unique<InvalidValue>(); +} + +} + +void +FieldValueNode::initFieldPath(const DocumentType& type) const { + if (_fieldPath.size() == 0) { + FieldPath::UP path(type.buildFieldPath(_fieldExpression)); + if (!path.get()) { + throw FieldNotFoundException( + vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'", + type.toString().c_str(), _fieldExpression.c_str()), + VESPA_STRLOC); + } + _fieldPath = *path; + } +} + +std::unique_ptr<Value> +FieldValueNode::getValue(const Context& context) const +{ + if (context._doc == NULL) { + return std::unique_ptr<Value>(new InvalidValue()); + } + + const Document& doc = *context._doc; + + if (!documentTypeEqualsName(doc.getType(), _doctype)) { + return std::unique_ptr<Value>(new InvalidValue()); + } + try{ + initFieldPath(doc.getType()); + + IteratorHandler handler; + doc.iterateNested(_fieldPath.getFullRange(), handler); + + if (handler.hasSingleValue()) { + return handler.getSingleValue(); + } else { + const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); + + if (values.size() == 0) { + return std::unique_ptr<Value>(new NullValue()); + } else { + return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); + } + } + } catch (vespalib::IllegalArgumentException& e) { + LOG(warning, "Caught exception while fetching field from document: %s", e.what()); + return std::unique_ptr<Value>(new InvalidValue()); + } catch (FieldNotFoundException& e) { + LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str()); + return std::unique_ptr<Value>(new InvalidValue()); + } +} + +void +FieldValueNode::visit(Visitor &visitor) const +{ + visitor.visitFieldValueNode(*this); +} + + +void +FieldValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _doctype << "." << _fieldExpression; + if (hadParentheses()) out << ')'; +} + + +std::unique_ptr<Value> +FieldValueNode::traceValue(const Context &context, std::ostream& out) const +{ + if (context._doc == NULL) { + return defaultTrace(getValue(context), out); + } + const Document &doc(*context._doc); + if (!documentTypeEqualsName(doc.getType(), _doctype)) { + out << "Document is of type " << doc.getType() << " which isn't a " + << _doctype << " document, thus resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue()); + } + try{ + initFieldPath(doc.getType()); + + IteratorHandler handler; + doc.iterateNested(_fieldPath.getFullRange(), handler); + + if (handler.hasSingleValue()) { + return handler.getSingleValue(); + } else { + const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); + + if (values.size() == 0) { + return std::unique_ptr<Value>(new NullValue()); + } else { + return std::unique_ptr<Value>(new ArrayValue(handler.getValues())); + } + } + } catch (FieldNotFoundException& e) { + LOG(warning, "Tried to compare to field %s, not found in document type", + _fieldExpression.c_str()); + out << "Field not found in document type " << doc.getType() + << ". Returning invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue()); + } +} + +IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, const vespalib::stringref & type, + int widthBits, int divisionBits) + : _bucketIdFactory(bucketIdFactory), + _id(name), + _typestring(type), + _type(ALL), + _widthBits(widthBits), + _divisionBits(divisionBits) +{ + if (type.length() > 2) switch (type[0]) { + case 'b': _type = BUCKET; + break; + case 'n': _type = NS; + break; + case 'g': + if (type[1] == 'r') { + _type = GROUP; + } else if (type[1] == 'i') { + _type = GID; + } + break; + case 's': { + if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; } + break; + } + case 't': + _type = TYPE; + break; + case 'u': + _type = USER; + break; + case 'o': + _type = ORDER; + break; + } +} + + +std::unique_ptr<Value> +IdValueNode::getValue(const Context& context) const +{ + if (context._doc != NULL) { + return getValue(context._doc->getId()); + } else if (context._docId != NULL) { + return getValue(*context._docId); + } else { + return getValue(context._docUpdate->getId()); + } +} + + +std::unique_ptr<Value> +IdValueNode::getValue(const DocumentId& id) const +{ + vespalib::string value; + switch (_type) { + case BUCKET: + return std::unique_ptr<Value>( + new IntegerValue( + _bucketIdFactory.getBucketId(id).getId(), true)); + case NS: + value = id.getScheme().getNamespace(); break; + case SCHEME: + value = id.getScheme().getTypeName(id.getScheme().getType()); + break; + case TYPE: + if (id.getScheme().hasDocType()) { + value = id.getScheme().getDocType(); + } else { + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case SPEC: + value = id.getScheme().getNamespaceSpecific(); + break; + case ALL: + value = id.getScheme().toString(); + break; + case GROUP: + if (id.getScheme().hasGroup()) { + value = id.getScheme().getGroup(); + } else { + fprintf(stderr, "***** Returning invalid value for %s\n", + id.toString().c_str()); + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case GID: + value = id.getGlobalId().toString(); + break; + case ORDER: + if (id.getScheme().getType() == IdString::ORDERDOC) { + const OrderDocIdString& ods( + static_cast<const OrderDocIdString&>(id.getScheme())); + if (ods.getWidthBits() == _widthBits + && ods.getDivisionBits() == _divisionBits) + { + return std::unique_ptr<Value>(new IntegerValue( + static_cast<const OrderDocIdString&>(id.getScheme()) + .getOrdering(), false)); + } + } + return std::unique_ptr<Value>(new InvalidValue()); + case USER: + if (id.getScheme().hasNumber()) { + return std::unique_ptr<Value>( + new IntegerValue(id.getScheme().getNumber(), false)); + } else { + return std::unique_ptr<Value>(new InvalidValue); + } + } + + return std::unique_ptr<Value>(new StringValue(value)); +} + + +std::unique_ptr<Value> +IdValueNode::traceValue(const Context& context, + std::ostream &out) const +{ + if (context._doc != NULL) { + return traceValue(context._doc->getId(), out); + } else if (context._docId != NULL) { + return traceValue(*context._docId, out); + } else { + return traceValue(context._docUpdate->getId(), out); + } +} + + +std::unique_ptr<Value> +IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const +{ + vespalib::string value; + switch (_type) { + case BUCKET: + { + document::BucketId bucket(_bucketIdFactory.getBucketId(id)); + std::unique_ptr<Value> result( + new IntegerValue(bucket.getId(), true)); + out << "Found id.bucket specification. Resolved to " + << bucket.toString() << ".\n"; + return result; + } + case NS: + value = id.getScheme().getNamespace(); + out << "Resolved id.namespace to value\"" << value << "\".\n"; + break; + case SCHEME: + value = id.getScheme().getTypeName(id.getScheme().getType()); + out << "Resolved id.scheme to value\"" << value << "\".\n"; + break; + case TYPE: + if (id.getScheme().hasDocType()) { + value = id.getScheme().getDocType(); + out << "Resolved id.type to value\"" << value << "\".\n"; + } else { + out << "Could not resolve type of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case SPEC: + value = id.getScheme().getNamespaceSpecific(); + out << "Resolved id.specific to value\"" << value << "\".\n"; + break; + case ALL: + value = id.getScheme().toString(); + out << "Resolved id to \"" << value << "\".\n"; + break; + case GROUP: + if (id.getScheme().hasGroup()) { + value = id.getScheme().getGroup(); + out << "Resolved group of doc (type " << id.getScheme().getType() + << ") to \"" << value << "\".\n"; + } else { + out << "Can't resolve group of doc \"" << id << "\".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + break; + case GID: + value = id.getGlobalId().toString(); + out << "Resolved gid to \"" << value << "\".\n"; + break; + case ORDER: + if (id.getScheme().getType() == IdString::ORDERDOC) { + const OrderDocIdString& ods( + static_cast<const OrderDocIdString&>(id.getScheme())); + if (ods.getWidthBits() == _widthBits + && ods.getDivisionBits() == _divisionBits) + { + std::unique_ptr<Value> result(new IntegerValue( + static_cast<const OrderDocIdString&>(id.getScheme()) + .getOrdering(), false)); + out << "Resolved id.order to int " << *result << "\n"; + return result; + } + } + out << "Could not resolve id.order(" << _widthBits << ", " + << _divisionBits << ") of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue()); + case USER: + if (id.getScheme().hasNumber()) { + std::unique_ptr<Value> result( + new IntegerValue(id.getScheme().getNumber(), false)); + out << "Resolved user of doc type " << id.getScheme().getType() + << " to " << *result << ".\n"; + return result; + } else { + out << "Could not resolve user of doc " << id << ".\n"; + return std::unique_ptr<Value>(new InvalidValue); + } + } + + return std::unique_ptr<Value>(new StringValue(value)); +} + + +void +IdValueNode::visit(Visitor &visitor) const +{ + visitor.visitIdValueNode(*this); +} + + +void +IdValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _id; + if (_type != ALL) { + out << '.' << _typestring; + } + if (_type == ORDER) { + out << "(" << _widthBits << "," << _divisionBits << ")"; + } + if (hadParentheses()) out << ')'; +} + +SearchColumnValueNode::SearchColumnValueNode( + const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, int numColumns) + : _bucketIdFactory(bucketIdFactory), + _id(name), + _numColumns(numColumns), + _distribution(std::make_unique<BucketDistribution>(_numColumns, 16)) +{ +} + +int64_t +SearchColumnValueNode::getValue(const BucketId& id) const +{ + return _distribution->getColumn(id); +} + + +std::unique_ptr<Value> +SearchColumnValueNode::getValue(const Context& context) const +{ + if (context._doc != NULL) { + return getValue(context._doc->getId()); + } else if (context._docId != NULL) { + return getValue(*context._docId); + } else { + return getValue(context._docUpdate->getId()); + } +} + + +std::unique_ptr<Value> +SearchColumnValueNode::getValue(const DocumentId& id) const +{ + return std::unique_ptr<Value>(new IntegerValue( + getValue(_bucketIdFactory.getBucketId(id)), false)); +} + + +std::unique_ptr<Value> +SearchColumnValueNode::traceValue(const Context& context, + std::ostream &out) const +{ + if (context._doc != NULL) { + return traceValue(context._doc->getId(), out); + } else if (context._docId != NULL) { + return traceValue(*context._docId, out); + } else { + return traceValue(context._docUpdate->getId(), out); + } +} + + +std::unique_ptr<Value> +SearchColumnValueNode::traceValue(const DocumentId& id, + std::ostream& out) const +{ + std::unique_ptr<Value> result(new IntegerValue( + getValue(_bucketIdFactory.getBucketId(id)), false)); + out << "Resolved search column of doc \"" << id << "\" to " << *result + << "\n"; + return result; +} + + +void +SearchColumnValueNode::visit(Visitor &visitor) const +{ + visitor.visitSearchColumnValueNode(*this); +} + + +void +SearchColumnValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + if (hadParentheses()) out << '('; + out << _id; + out << '.' << _numColumns; + if (hadParentheses()) out << ')'; +} + +namespace { + union HashUnion { + unsigned char _key[16]; + int64_t _hash[2]; + }; + int64_t hash(const void* data, uint32_t len) { + HashUnion hash; + fastc_md5sum((const unsigned char*) data, len, hash._key); + return hash._hash[0]; + } +} + +FunctionValueNode::FunctionValueNode(const vespalib::stringref & name, + std::unique_ptr<ValueNode> src) + : _function(), + _funcname(name), + _source(std::move(src)) +{ + if (name == "lowercase") { + _function = LOWERCASE; + } else if (name == "hash") { + _function = HASH; + } else if (name == "abs") { + _function = ABS; + } else { + throw ParsingFailedException("No function '"+name+"' exist.", + VESPA_STRLOC); + } +} + +std::unique_ptr<Value> +FunctionValueNode::getValue(std::unique_ptr<Value> val) const +{ + switch (val->getType()) { + case Value::String: + { + StringValue& sval(static_cast<StringValue&>(*val)); + if (_function == LOWERCASE) { + return std::unique_ptr<Value>(new StringValue( + vespalib::LowerCase::convert(sval.getValue()))); + } else if (_function == HASH) { + return std::unique_ptr<Value>(new IntegerValue( + hash(sval.getValue().c_str(), sval.getValue().size()), + false)); + } + break; + } + case Value::Float: + { + FloatValue& fval(static_cast<FloatValue&>(*val)); + if (_function == HASH) { + FloatValue::ValueType ffval = fval.getValue(); + return std::unique_ptr<Value>(new IntegerValue( + hash(&ffval, sizeof(ffval)), false)); + } else if (_function == ABS) { + FloatValue::ValueType ffval = fval.getValue(); + if (ffval < 0) ffval *= -1; + return std::unique_ptr<Value>(new FloatValue(ffval)); + } + break; + } + case Value::Integer: + { + IntegerValue& ival(static_cast<IntegerValue&>(*val)); + if (_function == HASH) { + IntegerValue::ValueType iival = ival.getValue(); + return std::unique_ptr<Value>(new IntegerValue( + hash(&iival, sizeof(iival)), false)); + } else if (_function == ABS) { + IntegerValue::ValueType iival = ival.getValue(); + if (iival < 0) iival *= -1; + return std::unique_ptr<Value>(new IntegerValue(iival, false)); + } + break; + } + case Value::Bucket: + { + throw ParsingFailedException( + "No functioncalls are allowed on value of type bucket", + VESPA_STRLOC); + break; + } + + case Value::Array: break; + case Value::Struct: break; + case Value::Invalid: break; + case Value::Null: break; + } + return std::unique_ptr<Value>(new InvalidValue); +} + +std::unique_ptr<Value> +FunctionValueNode::traceValue(std::unique_ptr<Value> val, + std::ostream& out) const +{ + switch (val->getType()) { + case Value::String: + { + StringValue& sval(static_cast<StringValue&>(*val)); + if (_function == LOWERCASE) { + std::unique_ptr<Value> result(new StringValue( + vespalib::LowerCase::convert(sval.getValue()))); + out << "Performed lowercase function on '" << sval + << "' => '" << *result << "'.\n"; + return result; + } else if (_function == HASH) { + std::unique_ptr<Value> result(new IntegerValue( + hash(sval.getValue().c_str(), sval.getValue().size()), + false)); + out << "Performed hash on string '" << sval << "' -> " + << *result << "\n"; + return result; + } + break; + } + case Value::Float: + { + FloatValue& fval(static_cast<FloatValue&>(*val)); + if (_function == HASH) { + FloatValue::ValueType ffval = fval.getValue(); + std::unique_ptr<Value> result(new IntegerValue( + hash(&ffval, sizeof(ffval)), false)); + out << "Performed hash on float " << ffval << " -> " << *result + << "\n"; + return result; + } else if (_function == ABS) { + FloatValue::ValueType ffval = fval.getValue(); + if (ffval < 0) ffval *= -1; + out << "Performed abs on float " << fval.getValue() << " -> " + << ffval << "\n"; + return std::unique_ptr<Value>(new FloatValue(ffval)); + } + break; + } + case Value::Integer: + { + IntegerValue& ival(static_cast<IntegerValue&>(*val)); + if (_function == HASH) { + IntegerValue::ValueType iival = ival.getValue(); + std::unique_ptr<Value> result(new IntegerValue( + hash(&iival, sizeof(iival)), false)); + out << "Performed hash on float " << iival << " -> " << *result + << "\n"; + return result; + } else if (_function == ABS) { + IntegerValue::ValueType iival = ival.getValue(); + if (iival < 0) iival *= -1; + out << "Performed abs on integer " << ival.getValue() << " -> " + << iival << "\n"; + return std::unique_ptr<Value>(new IntegerValue(iival, false)); + } + break; + } + case Value::Bucket: break; + case Value::Array: break; + case Value::Struct: break; + case Value::Invalid: break; + case Value::Null: break; + } + out << "Cannot use function " << _function << " on a value of type " + << val->getType() << ". Resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue); +} + + +void +FunctionValueNode::visit(Visitor &visitor) const +{ + visitor.visitFunctionValueNode(*this); +} + + +void +FunctionValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + if (hadParentheses()) out << '('; + _source->print(out, verbose, indent); + out << '.' << _funcname << "()"; + if (hadParentheses()) out << ')'; +} + +ArithmeticValueNode::ArithmeticValueNode( + std::unique_ptr<ValueNode> left, const vespalib::stringref & op, + std::unique_ptr<ValueNode> right) + : _operator(), + _left(std::move(left)), + _right(std::move(right)) +{ + if (op.size() == 1) switch (op[0]) { + case '+': _operator = ADD; return; + case '-': _operator = SUB; return; + case '*': _operator = MUL; return; + case '/': _operator = DIV; return; + case '%': _operator = MOD; return; + } + throw ParsingFailedException( + "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC); +} + +const char* +ArithmeticValueNode::getOperatorName() const +{ + switch (_operator) { + case ADD: return "+"; + case SUB: return "-"; + case MUL: return "*"; + case DIV: return "/"; + case MOD: return "%"; + } + return "UNKNOWN"; +} + + + +std::unique_ptr<Value> +ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval) const +{ + switch (_operator) { + case ADD: + { + if (lval->getType() == Value::String && + rval->getType() == Value::String) + { + StringValue& slval(static_cast<StringValue&>(*lval)); + StringValue& srval(static_cast<StringValue&>(*rval)); + return std::unique_ptr<Value>(new StringValue( + slval.getValue() + srval.getValue())); + } + } + case SUB: + case MUL: + case DIV: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + IntegerValue::ValueType res = 0; + switch (_operator) { + case ADD: res = ilval.getValue() + irval.getValue(); break; + case SUB: res = ilval.getValue() - irval.getValue(); break; + case MUL: res = ilval.getValue() * irval.getValue(); break; + case DIV: + if (irval.getValue() != 0) { + res = ilval.getValue() / irval.getValue(); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + break; + case MOD: assert(0); + } + return std::unique_ptr<Value>(new IntegerValue(res, false)); + } + NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); + NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get())); + if (nlval != 0 && nrval != 0) { + NumberValue::CommonValueType res = 0; + switch (_operator) { + case ADD: res = nlval->getCommonValue() + + nrval->getCommonValue(); break; + case SUB: res = nlval->getCommonValue() + - nrval->getCommonValue(); break; + case MUL: res = nlval->getCommonValue() + * nrval->getCommonValue(); break; + case DIV: + if (nrval->getCommonValue() != 0) { + res = nlval->getCommonValue() + / nrval->getCommonValue(); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + break; + case MOD: assert(0); + } + return std::unique_ptr<Value>(new FloatValue(res)); + } + } + case MOD: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + if (irval.getValue() != 0) { + return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false)); + } else { + throw vespalib::IllegalArgumentException("Division by zero"); + } + } + } + } + return std::unique_ptr<Value>(new InvalidValue); +} + +std::unique_ptr<Value> +ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval, + std::ostream& out) const +{ + switch (_operator) { + case ADD: + { + if (lval->getType() == Value::String && + rval->getType() == Value::String) + { + StringValue& slval(static_cast<StringValue&>(*lval)); + StringValue& srval(static_cast<StringValue&>(*rval)); + std::unique_ptr<Value> result(new StringValue( + slval.getValue() + srval.getValue())); + out << "Appended strings '" << slval << "' + '" << srval + << "' -> '" << *result << "'.\n"; + return result; + } + } + case SUB: + case MUL: + case DIV: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + IntegerValue::ValueType res = 0; + switch (_operator) { + case ADD: res = ilval.getValue() + irval.getValue(); break; + case SUB: res = ilval.getValue() - irval.getValue(); break; + case MUL: res = ilval.getValue() * irval.getValue(); break; + case DIV: res = ilval.getValue() / irval.getValue(); break; + case MOD: assert(0); + } + std::unique_ptr<Value> result(new IntegerValue(res, false)); + out << "Performed integer operation " << ilval << " " + << getOperatorName() << " " << irval << " = " << *result + << "\n"; + return result; + } + NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get())); + NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get())); + if (nlval != 0 && nrval != 0) { + NumberValue::CommonValueType res = 0; + switch (_operator) { + case ADD: res = nlval->getCommonValue() + + nrval->getCommonValue(); break; + case SUB: res = nlval->getCommonValue() + - nrval->getCommonValue(); break; + case MUL: res = nlval->getCommonValue() + * nrval->getCommonValue(); break; + case DIV: res = nlval->getCommonValue() + / nrval->getCommonValue(); break; + case MOD: assert(0); + } + std::unique_ptr<Value> result(new FloatValue(res)); + out << "Performed float operation " << nlval << " " + << getOperatorName() << " " << nrval << " = " << *result + << "\n"; + return result; + } + } + case MOD: + { + if (lval->getType() == Value::Integer && + rval->getType() == Value::Integer) + { + IntegerValue& ilval(static_cast<IntegerValue&>(*lval)); + IntegerValue& irval(static_cast<IntegerValue&>(*rval)); + std::unique_ptr<Value> result(new IntegerValue( + ilval.getValue() % irval.getValue(), false)); + out << "Performed integer operation " << ilval << " " + << getOperatorName() << " " << irval << " = " << *result + << "\n"; + return result; + } + } + } + out << "Failed to do operation " << getOperatorName() + << " on values of type " << lval->getType() << " and " + << rval->getType() << ". Resolving invalid.\n"; + return std::unique_ptr<Value>(new InvalidValue); +} + + +void +ArithmeticValueNode::visit(Visitor &visitor) const +{ + visitor.visitArithmeticValueNode(*this); +} + + +void +ArithmeticValueNode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + if (hadParentheses()) out << '('; + _left->print(out, verbose, indent); + switch (_operator) { + case ADD: out << " + "; break; + case SUB: out << " - "; break; + case MUL: out << " * "; break; + case DIV: out << " / "; break; + case MOD: out << " % "; break; + } + _right->print(out, verbose, indent); + if (hadParentheses()) out << ')'; +} + +} + diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h new file mode 100644 index 00000000000..06d92190943 --- /dev/null +++ b/document/src/vespa/document/select/valuenodes.h @@ -0,0 +1,338 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "valuenode.h" +#include <vespa/document/base/fieldpath.h> + +namespace document { + +class BucketDistribution; +class BucketIdFactory; +class DocumentId; +class BucketId; +class DocumentType; + +namespace select { + +class InvalidValueNode : public ValueNode +{ + vespalib::string _name; +public: + InvalidValueNode(const vespalib::stringref & name); + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new InvalidValue()); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new InvalidValueNode(_name)); + } +}; + +class NullValueNode : public ValueNode +{ + vespalib::string _name; +public: + NullValueNode(const vespalib::stringref & name); + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new NullValue()); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new NullValueNode(_name)); + } +}; + +class StringValueNode : public ValueNode +{ + vespalib::string _value; +public: + StringValueNode(const vespalib::stringref & val); + + const vespalib::string& getValue() const { return _value; } + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new StringValue(_value)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new StringValueNode(_value)); + } +}; + +class IntegerValueNode : public ValueNode +{ + int64_t _value; + bool _isBucketValue; +public: + IntegerValueNode(int64_t val, bool isBucketValue) + : _value(val), _isBucketValue(isBucketValue) {} + + int64_t getValue() const { return _value; } + + virtual std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new IntegerValueNode(_value, _isBucketValue)); + } +}; + +class CurrentTimeValueNode : public ValueNode +{ +public: + int64_t getValue() const; + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new IntegerValue(getValue(), false)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new CurrentTimeValueNode); + } +}; + +class VariableValueNode : public ValueNode +{ + vespalib::string _value; +public: + VariableValueNode(const vespalib::string & variableName) : _value(variableName) {} + + const vespalib::string& getVariableName() const { return _value; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new VariableValueNode(_value)); + } +}; + +class FloatValueNode : public ValueNode +{ + double _value; +public: + FloatValueNode(double val) : _value(val) {} + + double getValue() const { return _value; } + + std::unique_ptr<Value> getValue(const Context&) const override { + return std::unique_ptr<Value>(new FloatValue(_value)); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FloatValueNode(_value)); + } +}; + +class FieldValueNode : public ValueNode +{ + vespalib::string _doctype; + vespalib::string _fieldExpression; + vespalib::string _fieldName; + mutable FieldPath _fieldPath; + +public: + FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression); + FieldValueNode(const FieldValueNode &); + FieldValueNode & operator = (const FieldValueNode &); + FieldValueNode(FieldValueNode &&) = default; + FieldValueNode & operator = (FieldValueNode &&) = default; + ~FieldValueNode(); + + const vespalib::string& getDocType() const { return _doctype; } + const vespalib::string& getRealFieldName() const { return _fieldName; } + const vespalib::string& getFieldName() const { return _fieldExpression; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); + } + + static vespalib::string extractFieldName(const std::string & fieldExpression); + +private: + + void initFieldPath(const DocumentType&) const; +}; + +class IdValueNode : public ValueNode +{ +public: + enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL }; + + IdValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, const vespalib::stringref & type, + int widthBits = -1, int divisionBits = -1); + + Type getType() const { return _type; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + + std::unique_ptr<Value> getValue(const DocumentId& id) const; + + std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; + + std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); + } + + int getWidthBits() const { return _widthBits; } + int getDivisionBits() const { return _divisionBits; } + +private: + const BucketIdFactory& _bucketIdFactory; + vespalib::string _id; + vespalib::string _typestring; + Type _type; + int _widthBits; + int _divisionBits; +}; + +class SearchColumnValueNode : public ValueNode +{ +public: + SearchColumnValueNode(const BucketIdFactory& bucketIdFactory, + const vespalib::stringref & name, + int numColumns); + + int getColumns() { return _numColumns; } + + std::unique_ptr<Value> getValue(const Context& context) const override; + std::unique_ptr<Value> getValue(const DocumentId& id) const; + std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override; + std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const; + + int64_t getValue(const BucketId& bucketId) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns)); +} + +private: + const BucketIdFactory& _bucketIdFactory; + vespalib::string _id; + int _numColumns; + std::unique_ptr<BucketDistribution> _distribution; +}; + +class FunctionValueNode : public ValueNode +{ +public: + enum Function { LOWERCASE, HASH, ABS }; + + FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src); + + Function getFunction() const { return _function; } + const vespalib::string &getFunctionName(void) const { return _funcname; } + + std::unique_ptr<Value> getValue(const Context& context) const override { + return getValue(_source->getValue(context)); + } + + std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override { + return traceValue(_source->getValue(context), out); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new FunctionValueNode(_funcname, _source->clone())); + } + + const ValueNode& getChild() const { return *_source; } + +private: + Function _function; + vespalib::string _funcname; + std::unique_ptr<ValueNode> _source; + + virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const; + virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val, + std::ostream& out) const; +}; + +class ArithmeticValueNode : public ValueNode +{ +public: + enum Operator { ADD, SUB, MUL, DIV, MOD }; + + ArithmeticValueNode(std::unique_ptr<ValueNode> left, + const vespalib::stringref & op, + std::unique_ptr<ValueNode> right); + + Operator getOperator() const { return _operator; } + const char* getOperatorName() const; + + std::unique_ptr<Value> + getValue(const Context& context) const override { + return getValue(_left->getValue(context), _right->getValue(context)); + } + + std::unique_ptr<Value> + traceValue(const Context &context, std::ostream& out) const override { + return traceValue(_left->getValue(context), _right->getValue(context), out); + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + ValueNode::UP clone() const override { + return wrapParens(new ArithmeticValueNode(_left->clone(), + getOperatorName(), + _right->clone())); + } + + const ValueNode& getLeft() const { return *_left; } + const ValueNode& getRight() const { return *_right; } + +private: + Operator _operator; + std::unique_ptr<ValueNode> _left; + std::unique_ptr<ValueNode> _right; + + virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval) const; + virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval, + std::unique_ptr<Value> rval, + std::ostream&) const; +}; + +} // select +} // document diff --git a/document/src/vespa/document/select/variablemap.h b/document/src/vespa/document/select/variablemap.h new file mode 100644 index 00000000000..86a1cb85e63 --- /dev/null +++ b/document/src/vespa/document/select/variablemap.h @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/string.h> + +namespace document::select { + +using VariableMapT = vespalib::hash_map<vespalib::string, double>; + +class VariableMap : public VariableMapT { +public: + using VariableMapT::VariableMapT; +}; + +} diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 93856b376aa..b2a2bd098a6 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -27,6 +27,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/log/log.h> LOG_SETUP(".vespadocumentdeserializer"); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 11317f2ff0a..1f6a4a4f0b0 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -24,7 +24,9 @@ #include <vespa/document/update/fieldpathupdates.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> +#include <vespa/document/util/compressor.h> using std::make_pair; using std::pair; diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp index 03a31d882f3..0f8b730367e 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.cpp +++ b/document/src/vespa/document/update/addfieldpathupdate.cpp @@ -1,27 +1,24 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "addfieldpathupdate.h" -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/select/parser.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <ostream> using vespalib::nbostream; -LOG_SETUP(".document.update.fieldpathupdate"); - namespace document { +using namespace fieldvalue; +using vespalib::make_string; + IMPLEMENT_IDENTIFIABLE(AddFieldPathUpdate, FieldPathUpdate); -AddFieldPathUpdate::AddFieldPathUpdate( - const DocumentTypeRepo& repo, - const DataType& type, - stringref fieldPath, - stringref whereClause, - const ArrayFieldValue& values) +AddFieldPathUpdate::AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type, + stringref fieldPath, stringref whereClause, const ArrayFieldValue& values) : FieldPathUpdate(repo, type, fieldPath, whereClause), _values(vespalib::CloneablePtr<ArrayFieldValue>(values.clone())) { @@ -39,22 +36,33 @@ AddFieldPathUpdate::clone() const { return new AddFieldPathUpdate(*this); } -FieldValue::IteratorHandler::ModificationStatus -AddFieldPathUpdate::AddIteratorHandler::doModify(FieldValue& fv) -{ - LOG(spam, "Adding values to %s", fv.toString().c_str()); +namespace { + +class AddIteratorHandler : public fieldvalue::IteratorHandler { +public: + AddIteratorHandler(const ArrayFieldValue &values) : _values(values) {} + fieldvalue::ModificationStatus doModify(FieldValue &fv) override; + bool createMissingPath() const override { return true; } + bool onComplex(const fieldvalue::IteratorHandler::Content &) override { return false; } +private: + const ArrayFieldValue &_values; +}; + + +ModificationStatus +AddIteratorHandler::doModify(FieldValue &fv) { if (fv.inherits(CollectionFieldValue::classId)) { - CollectionFieldValue& cf = static_cast<CollectionFieldValue&>(fv); + CollectionFieldValue &cf = static_cast<CollectionFieldValue &>(fv); for (std::size_t i = 0; i < _values.size(); ++i) { cf.add(_values[i]); } } else { - vespalib::string err = vespalib::make_string( - "Unable to add a value to a \"%s\" field value.", - fv.getClass().name()); - throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); + vespalib::string err = make_string("Unable to add a value to a \"%s\" field value.", fv.getClass().name()); + throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); } - return MODIFIED; + return ModificationStatus::MODIFIED; +} + } bool @@ -68,8 +76,7 @@ AddFieldPathUpdate::operator==(const FieldPathUpdate& other) const } void -AddFieldPathUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +AddFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "AddFieldPathUpdate(\n"; FieldPathUpdate::print(out, verbose, indent + " "); @@ -79,8 +86,7 @@ AddFieldPathUpdate::print(std::ostream& out, bool verbose, } void -AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, - const DataType& type, +AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) { FieldPathUpdate::deserialize(repo, type, buffer, version); @@ -95,4 +101,11 @@ AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, buffer.incPos(buffer.getRemaining() - stream.size()); } +std::unique_ptr<IteratorHandler> +AddFieldPathUpdate::getIteratorHandler(Document&) const { + return std::make_unique<AddIteratorHandler>(*_values); +} + + + } // ns document diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h index 6d75ebc4209..94156156257 100644 --- a/document/src/vespa/document/update/addfieldpathupdate.h +++ b/document/src/vespa/document/update/addfieldpathupdate.h @@ -10,7 +10,6 @@ class AddFieldPathUpdate : public FieldPathUpdate public: /** For deserialization */ AddFieldPathUpdate(); - AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type, stringref fieldPath, stringref whereClause, const ArrayFieldValue& values); ~AddFieldPathUpdate(); @@ -18,7 +17,6 @@ public: FieldPathUpdate* clone() const override; bool operator==(const FieldPathUpdate& other) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - const ArrayFieldValue & getValues() const { return *_values; } DECLARE_IDENTIFIABLE(AddFieldPathUpdate); @@ -29,22 +27,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class AddIteratorHandler : public FieldValue::IteratorHandler - { - public: - AddIteratorHandler(const ArrayFieldValue& values) : _values(values) { } - - ModificationStatus doModify(FieldValue& fv) override; - bool createMissingPath() const override { return true; } - bool onComplex(const Content&) override { return false; } - private: - const ArrayFieldValue& _values; - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AddIteratorHandler(*_values)); - } + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override; vespalib::CloneablePtr<ArrayFieldValue> _values; }; diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index 910e3d7dd50..b1142836fc2 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -2,17 +2,17 @@ #include "addvalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/datatype/arraydatatype.h> -#include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/util/serializable.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/xmlstream.h> + using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index 29c24e7140f..a8fe541d0fe 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -3,9 +3,11 @@ #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index 0a676c88df2..7ab0a43dbe1 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -1,9 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "assignfieldpathupdate.h" #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/parser.h> +#include <vespa/document/select/variablemap.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/update/assignfieldpathupdate.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/numeric/conversion/cast.hpp> @@ -15,6 +17,8 @@ using vespalib::nbostream; namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE(AssignFieldPathUpdate, FieldPathUpdate); AssignFieldPathUpdate::AssignFieldPathUpdate() @@ -68,24 +72,56 @@ FieldPathUpdate* AssignFieldPathUpdate::clone() const { return new AssignFieldPathUpdate(*this); } +namespace { -std::unique_ptr<FieldValue::IteratorHandler> -AssignFieldPathUpdate::getIteratorHandler(Document& doc) const +class AssignValueIteratorHandler : public IteratorHandler { - if (!_expression.empty()) { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AssignExpressionIteratorHandler( - *_repo, doc, _expression, _removeIfZero, _createMissingPath)); - } else { - return std::unique_ptr<FieldValue::IteratorHandler>( - new AssignValueIteratorHandler( - *_newValue, _removeIfZero, _createMissingPath)); - } -} - - -FieldValue::IteratorHandler::ModificationStatus -AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) { +public: + AssignValueIteratorHandler(const FieldValue& newValue, + bool removeIfZero, + bool createMissingPath_) + : _newValue(newValue), _removeIfZero(removeIfZero), + _createMissingPath(createMissingPath_) + {} + + ModificationStatus doModify(FieldValue& fv) override; + bool onComplex(const Content&) override { return false; } + bool createMissingPath() const override { return _createMissingPath; } + +private: + const FieldValue& _newValue; + bool _removeIfZero; + bool _createMissingPath; +}; + +class AssignExpressionIteratorHandler : public IteratorHandler +{ +public: + AssignExpressionIteratorHandler( + const DocumentTypeRepo& repo, + Document& doc, + const vespalib::string& expression, + bool removeIfZero, + bool createMissingPath_) + : _calc(repo, expression), + _doc(doc), + _removeIfZero(removeIfZero), + _createMissingPath(createMissingPath_) + {} + + ModificationStatus doModify(FieldValue& fv) override; + bool onComplex(const Content&) override { return false; } + bool createMissingPath() const override { return _createMissingPath; } + +private: + DocumentCalculator _calc; + Document& _doc; + bool _removeIfZero; + bool _createMissingPath; +}; + +ModificationStatus +AssignValueIteratorHandler::doModify(FieldValue& fv) { LOG(spam, "fv = %s", fv.toString().c_str()); if (!(*fv.getDataType() == *_newValue.getDataType())) { std::string err = vespalib::make_string( @@ -98,17 +134,18 @@ AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) { && _newValue.inherits(NumericFieldValueBase::classId) && static_cast<const NumericFieldValueBase&>(_newValue).getAsLong() == 0) { - return REMOVED; + return ModificationStatus::REMOVED; } fv.assign(_newValue); - return MODIFIED; + return ModificationStatus::MODIFIED; } -FieldValue::IteratorHandler::ModificationStatus -AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) { +ModificationStatus +AssignExpressionIteratorHandler::doModify(FieldValue& fv) { LOG(spam, "fv = %s", fv.toString().c_str()); if (fv.inherits(NumericFieldValueBase::classId)) { - DocumentCalculator::VariableMap vars; + std::unique_ptr<select::VariableMap> varHolder = std::make_unique<select::VariableMap>(); + select::VariableMap & vars = *varHolder; for (VariableMap::const_iterator i(getVariables().begin()), e(getVariables().end()); i != e; ++i) { @@ -122,18 +159,18 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) vars["value"] = fv.getAsDouble(); try { - double res = _calc.evaluate(_doc, std::move(vars)); + double res = _calc.evaluate(_doc, std::move(varHolder)); if (_removeIfZero && static_cast<uint64_t>(res) == 0) { - return REMOVED; + return ModificationStatus::REMOVED; } else { fv.assign(DoubleFieldValue(res)); } } catch (const vespalib::IllegalArgumentException&) { // Divide by zero does not modify the document field - return NOT_MODIFIED; + return ModificationStatus::NOT_MODIFIED; } catch (const boost::bad_numeric_cast&) { // Underflow/overflow does not modify - return NOT_MODIFIED; + return ModificationStatus::NOT_MODIFIED; } } else { throw vespalib::IllegalArgumentException( @@ -141,7 +178,19 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) fv.toString().c_str(), fv.getDataType()->toString().c_str()), VESPA_STRLOC); } - return MODIFIED; + return ModificationStatus::MODIFIED; +} + +} + +std::unique_ptr<IteratorHandler> +AssignFieldPathUpdate::getIteratorHandler(Document& doc) const +{ + if (!_expression.empty()) { + return std::make_unique<AssignExpressionIteratorHandler>(*_repo, doc, _expression, _removeIfZero, _createMissingPath); + } else { + return std::make_unique<AssignValueIteratorHandler>(*_newValue, _removeIfZero, _createMissingPath); + } } bool diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index 81853246e54..3794cb331ce 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -58,53 +58,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class AssignValueIteratorHandler : public FieldValue::IteratorHandler - { - public: - AssignValueIteratorHandler(const FieldValue& newValue, - bool removeIfZero, - bool createMissingPath_) - : _newValue(newValue), _removeIfZero(removeIfZero), - _createMissingPath(createMissingPath_) - {} - - ModificationStatus doModify(FieldValue& fv) override; - bool onComplex(const Content&) override { return false; } - bool createMissingPath() const override { return _createMissingPath; } - - private: - const FieldValue& _newValue; - bool _removeIfZero; - bool _createMissingPath; - }; - - class AssignExpressionIteratorHandler : public FieldValue::IteratorHandler - { - public: - AssignExpressionIteratorHandler( - const DocumentTypeRepo& repo, - Document& doc, - const vespalib::string& expression, - bool removeIfZero, - bool createMissingPath_) - : _calc(repo, expression), - _doc(doc), - _removeIfZero(removeIfZero), - _createMissingPath(createMissingPath_) - {} - - ModificationStatus doModify(FieldValue& fv) override; - bool onComplex(const Content&) override { return false; } - bool createMissingPath() const override { return _createMissingPath; } - - private: - DocumentCalculator _calc; - Document& _doc; - bool _removeIfZero; - bool _createMissingPath; - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document& doc) const override; + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const override; const DocumentTypeRepo *_repo; FieldValue::CP _newValue; diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp index 0538bec10e5..63609ed18c1 100644 --- a/document/src/vespa/document/update/assignvalueupdate.cpp +++ b/document/src/vespa/document/update/assignvalueupdate.cpp @@ -3,14 +3,16 @@ #include "assignvalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> + using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp index 13143e004ee..f34987ccfeb 100644 --- a/document/src/vespa/document/update/clearvalueupdate.cpp +++ b/document/src/vespa/document/update/clearvalueupdate.cpp @@ -1,15 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "clearvalueupdate.h" #include <vespa/document/base/field.h> -#include <vespa/document/update/clearvalueupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; +using namespace vespalib::xml; -namespace document -{ +namespace document { IMPLEMENT_IDENTIFIABLE(ClearValueUpdate, ValueUpdate); diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index 866a215505c..43fe6c28dc8 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "documentupdate.h" #include "documentupdateflags.h" #include <vespa/document/fieldvalue/fieldvalues.h> @@ -7,12 +8,15 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bufferexceptions.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; using vespalib::make_string; using vespalib::string; +using namespace vespalib::xml; namespace document { @@ -103,7 +107,28 @@ DocumentUpdate::affectsDocumentBody() const return false; } -// Print the content of this document update. +const DocumentType& +DocumentUpdate::getType() const { + return static_cast<const DocumentType &> (*_type); +} + +DocumentUpdate& +DocumentUpdate::addUpdate(const FieldUpdate& update) { + _updates.push_back(update); + return *this; +} + +DocumentUpdate& +DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) { + _fieldPathUpdates.push_back(update); + return *this; +} + +DocumentUpdate* +DocumentUpdate::clone() const { + return new DocumentUpdate(*this); +} + void DocumentUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -230,8 +255,7 @@ DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, ByteBuffer& buffer) try{ buffer.getShortNetwork(_version); - std::pair<const DocumentType *, DocumentId> typeAndId( - deserializeTypeAndId(repo, buffer)); + std::pair<const DocumentType *, DocumentId> typeAndId(deserializeTypeAndId(repo, buffer)); _type = typeAndId.first; _documentId = typeAndId.second; // Read field updates, if any. diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index 5d09bd36816..a1d3e910cb5 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -29,7 +29,6 @@ #include "fieldpathupdate.h" #include <vespa/document/base/documentid.h> #include <vespa/document/base/field.h> -#include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/document/util/bytebuffer.h> @@ -118,19 +117,13 @@ public: * Add a field update to this document update. * @return A reference to this. */ - DocumentUpdate& addUpdate(const FieldUpdate& update) { - _updates.push_back(update); - return *this; - } + DocumentUpdate& addUpdate(const FieldUpdate& update); /** * Add a fieldpath update to this document update. * @return A reference to this. */ - DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update) { - _fieldPathUpdates.push_back(update); - return *this; - } + DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update); /** @return The list of updates. */ const FieldUpdateV & getUpdates() const { return _updates; } @@ -141,9 +134,8 @@ public: bool affectsDocumentBody() const; /** @return The type of document this update is for. */ - const DocumentType& getType() const { return static_cast<const DocumentType &> (*_type); } + const DocumentType& getType() const; - // Printable implementation void print(std::ostream& out, bool verbose, const std::string& indent) const override; void deserialize42(const DocumentTypeRepo&, ByteBuffer&); @@ -155,13 +147,9 @@ public: void serialize42(vespalib::nbostream &stream) const; void serializeHEAD(vespalib::nbostream &stream) const; - // XmlSerializable implementation void printXml(XmlOutputStream&) const override; - // Cloneable implementation - virtual DocumentUpdate* clone() const { - return new DocumentUpdate(*this); - } + virtual DocumentUpdate* clone() const; /** * Sets whether this update should create the document it updates if that document does not exist. @@ -199,7 +187,6 @@ private: DocumentUpdate(); int deserializeFlags(int sizeAndFlags); - }; } // document diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index a4145a9b58a..11ed5f08c93 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -1,14 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/fieldvalues.h> +#include "fieldpathupdates.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/select/parser.h> -#include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/log/log.h> +#include <ostream> +#include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); +using vespalib::make_string; +using vespalib::IllegalArgumentException; + namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldPathUpdate, Identifiable); namespace { @@ -43,9 +50,9 @@ FieldPathUpdate::FieldPathUpdate(const DocumentTypeRepo& repo, : std::unique_ptr<select::Node>()) { if (!_fieldPath.get()) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Could not create field path update for: path='%s', where='%s'", - fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC); + throw IllegalArgumentException( + make_string("Could not create field path update for: path='%s', where='%s'", + fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC); } } @@ -61,7 +68,7 @@ FieldPathUpdate::operator==(const FieldPathUpdate& other) const void FieldPathUpdate::applyTo(Document& doc) const { - std::unique_ptr<FieldValue::IteratorHandler> handler(getIteratorHandler(doc)); + std::unique_ptr<IteratorHandler> handler(getIteratorHandler(doc)); if (!_whereClause) { doc.iterateNested(*_fieldPath, *handler); @@ -70,7 +77,7 @@ FieldPathUpdate::applyTo(Document& doc) const for (select::ResultList::const_iterator i = results.begin(); i != results.end(); ++i) { - LOG(spam, "vars = %s", FieldValue::IteratorHandler::toString(handler->getVariables()).c_str()); + LOG(spam, "vars = %s", handler->getVariables().toString().c_str()); if (*i->second == select::Result::True) { handler->setVariables(i->first); doc.iterateNested(*_fieldPath, *handler); @@ -88,8 +95,7 @@ FieldPathUpdate::affectsDocumentBody() const } void -FieldPathUpdate::print(std::ostream& out, bool, - const std::string& indent) const +FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const { out << indent << "fieldPath='" << _originalFieldPath << "',\n" << indent << "whereClause='" << _originalWhereClause << "'"; @@ -99,10 +105,10 @@ void FieldPathUpdate::checkCompatibility(const FieldValue& fv) const { if ( !getResultingDataType().isValueType(fv)) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Cannot update a '%s' field with a '%s' value", - getResultingDataType().toString().c_str(), - fv.getDataType()->toString().c_str()), + throw IllegalArgumentException( + make_string("Cannot update a '%s' field with a '%s' value", + getResultingDataType().toString().c_str(), + fv.getDataType()->toString().c_str()), VESPA_STRLOC); } } @@ -111,8 +117,7 @@ const DataType& FieldPathUpdate::getResultingDataType() const { if (_fieldPath->empty()) { - throw vespalib::IllegalStateException("Cannot get resulting data " - "type from an empty field path", VESPA_STRLOC); + throw vespalib::IllegalStateException("Cannot get resulting data type from an empty field path", VESPA_STRLOC); } return _fieldPath->rbegin()->getDataType(); } @@ -138,9 +143,7 @@ FieldPathUpdate::deserialize(const DocumentTypeRepo& repo, try { _fieldPath = type.buildFieldPath(_originalFieldPath).release(); if (!_fieldPath.get()) { - throw DeserializeException( - vespalib::make_string("Invalid field path: '%s'", _originalFieldPath.c_str()), - VESPA_STRLOC); + throw DeserializeException(make_string("Invalid field path: '%s'", _originalFieldPath.c_str()), VESPA_STRLOC); } _whereClause = !_originalWhereClause.empty() ? parseDocumentSelection(_originalWhereClause, repo) @@ -159,8 +162,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, buffer.getByte(updateType); std::unique_ptr<FieldPathUpdate> update; - switch (updateType) - { + switch (updateType) { case 0: update.reset(new AssignFieldPathUpdate()); break; @@ -171,9 +173,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo, update.reset(new AddFieldPathUpdate()); break; default: - throw DeserializeException( - vespalib::make_string("Unknown fieldpath update type: %d", updateType), - VESPA_STRLOC); + throw DeserializeException(make_string("Unknown fieldpath update type: %d", updateType), VESPA_STRLOC); } update->deserialize(repo, type, buffer, serializationVersion); return update; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 80a63d62281..0eaff9485bf 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -104,8 +104,7 @@ protected: enum SerializedMagic {AssignMagic=0, RemoveMagic=1, AddMagic=2}; private: // TODO: rename to createIteratorHandler? - virtual std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler( - Document& doc) const = 0; + virtual std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const = 0; vespalib::string _originalFieldPath; vespalib::string _originalWhereClause; diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index 3141fdf3a4f..e25b95e56c1 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -3,6 +3,7 @@ #include "fieldupdate.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> namespace document { diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index 0a92fa7fb15..37d775bb204 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -5,13 +5,14 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; -namespace document -{ +namespace document { IMPLEMENT_IDENTIFIABLE(MapValueUpdate, ValueUpdate); diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp index 6c9d60abd2f..4af867995d9 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.cpp +++ b/document/src/vespa/document/update/removefieldpathupdate.cpp @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "removefieldpathupdate.h" -#include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/select/parser.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> +#include <ostream> namespace document { +using namespace fieldvalue; + IMPLEMENT_IDENTIFIABLE(RemoveFieldPathUpdate, FieldPathUpdate); RemoveFieldPathUpdate::RemoveFieldPathUpdate() @@ -30,8 +32,7 @@ RemoveFieldPathUpdate::operator==(const FieldPathUpdate& other) const } void -RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "RemoveFieldPathUpdate(\n"; FieldPathUpdate::print(out, verbose, indent + " "); @@ -39,11 +40,28 @@ RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, } void -RemoveFieldPathUpdate::deserialize( - const DocumentTypeRepo& repo, const DataType& type, - ByteBuffer& buffer, uint16_t version) +RemoveFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type, + ByteBuffer& buffer, uint16_t version) { FieldPathUpdate::deserialize(repo, type, buffer, version); } +namespace { + +class RemoveIteratorHandler : public IteratorHandler { +public: + RemoveIteratorHandler() {} + + ModificationStatus doModify(FieldValue &) override { + return ModificationStatus::REMOVED; + } +}; + +} + +std::unique_ptr<IteratorHandler> +RemoveFieldPathUpdate::getIteratorHandler(Document&) const { + return std::make_unique<RemoveIteratorHandler>(); +} + } // ns document diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h index 6de2a105cbf..1a3865fe761 100644 --- a/document/src/vespa/document/update/removefieldpathupdate.h +++ b/document/src/vespa/document/update/removefieldpathupdate.h @@ -29,19 +29,7 @@ private: void deserialize(const DocumentTypeRepo& repo, const DataType& type, ByteBuffer& buffer, uint16_t version) override; - class RemoveIteratorHandler : public FieldValue::IteratorHandler - { - public: - RemoveIteratorHandler() {} - - ModificationStatus doModify(FieldValue&) override { - return REMOVED; - } - }; - - std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override { - return std::make_unique<RemoveIteratorHandler>(); - } + std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override; }; diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index e0575f600da..34af57edbe8 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -1,20 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "removevalueupdate.h" #include <vespa/document/base/field.h> #include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/fieldvalue/fieldvalues.h> -#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> -#include <vespa/document/update/removevalueupdate.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> +#include <vespa/vespalib/util/xmlstream.h> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; using vespalib::nbostream; +using namespace vespalib::xml; -namespace document -{ +namespace document { IMPLEMENT_IDENTIFIABLE(RemoveValueUpdate, ValueUpdate); diff --git a/document/src/vespa/document/util/xmlserializable.h b/document/src/vespa/document/util/xmlserializable.h index 21e0b858dec..f74761a04f6 100644 --- a/document/src/vespa/document/util/xmlserializable.h +++ b/document/src/vespa/document/util/xmlserializable.h @@ -5,14 +5,6 @@ #include <vespa/vespalib/util/xmlserializable.h> namespace document { - typedef vespalib::xml::XmlOutputStream XmlOutputStream; - typedef vespalib::xml::XmlSerializable XmlSerializable; - typedef vespalib::xml::XmlTag XmlTag; - typedef vespalib::xml::XmlEndTag XmlEndTag; - typedef vespalib::xml::XmlAttribute XmlAttribute; - typedef vespalib::xml::XmlContent XmlContent; - typedef vespalib::xml::XmlEscapedContent XmlEscapedContent; - typedef vespalib::xml::XmlBase64Content XmlBase64Content; - typedef vespalib::xml::XmlContentWrapper XmlContentWrapper; + using XmlSerializable = vespalib::xml::XmlSerializable; + using XmlOutputStream = vespalib::xml::XmlOutputStream; } - diff --git a/documentapi/OWNERS b/documentapi/OWNERS index 123437e2758..e030acdbc5b 100644 --- a/documentapi/OWNERS +++ b/documentapi/OWNERS @@ -1 +1 @@ -dybis +freva diff --git a/documentapi/pom.xml b/documentapi/pom.xml index 17fd0cc5505..41552fb33e1 100644 --- a/documentapi/pom.xml +++ b/documentapi/pom.xml @@ -43,6 +43,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> <version>${project.version}</version> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 793e0ed73c0..9f20ca78d6e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -198,15 +198,6 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { return errors.toString(); } - static Set<Integer> - getErrorCodes(Reply reply) { - Set<Integer> errorCodes = new HashSet<>(); - for (int i = 0; i < reply.getNumErrors(); ++i) { - errorCodes.add(reply.getError(i).getCode()); - } - return errorCodes; - } - private static Result.ResultType messageBusErrorToResultType(int messageBusError) { switch (messageBusError) { case ErrorCode.SEND_QUEUE_FULL: return Result.ResultType.TRANSIENT_ERROR; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 106791491f9..47eeb12b250 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -167,8 +167,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re msg.setPriority(pri); Reply reply = syncSend(msg); if (reply.hasErrors()) { - throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), - MessageBusAsyncSession.getErrorCodes(reply)); + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); } if (reply.getType() != DocumentProtocol.REPLY_UPDATEDOCUMENT) { throw new DocumentAccessException("Received unknown response: " + reply); @@ -218,8 +217,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) { Reply reply = syncSend(putDocumentMessage); if (reply.hasErrors()) { - throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), - MessageBusAsyncSession.getErrorCodes(reply)); + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); } } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java index 26b0eec8cf4..1a3a23eb8e4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -55,22 +55,18 @@ class RoutingPolicyRepository { log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'."); return null; } - DocumentProtocolRoutingPolicy ret; - try { - ret = factory.createPolicy(param); - } catch (Exception e) { - ret = new ErrorPolicy(e.getMessage()); + final DocumentProtocolRoutingPolicy ret = factory.createPolicy(param); + + if (ret == null) { + log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + + "routing policy for parameter '" + name + "'."); + return null; } if (ret.getMetrics() != null) { metrics.routingPolicyMetrics.addMetric(ret.getMetrics()); } - if (ret == null) { - log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + - "routing policy for parameter '" + name + "'."); - return null; - } return ret; } } diff --git a/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore b/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore +++ /dev/null diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java new file mode 100644 index 00000000000..609e83683b1 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepositoryTest.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RoutingPolicyRepositoryTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void policy_creation_does_not_swallow_exception() { + final DocumentProtocolMetricSet metrics = new DocumentProtocolMetricSet(); + final RoutingPolicyRepository repo = new RoutingPolicyRepository(metrics); + final RoutingPolicyFactory factory = mock(RoutingPolicyFactory.class); + + when(factory.createPolicy(anyString())).thenThrow(new IllegalArgumentException("oh no!")); + repo.putFactory("foo", factory); + + expectedException.expectMessage("oh no!"); + expectedException.expect(IllegalArgumentException.class); + + repo.createPolicy("foo", "bar"); + } + +} diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index 58fc96b2e78..e4b04b0afc7 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocrepo.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 44aea1cb169..834bea0e757 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages50test.h" -#include <vespa/document/datatype/datatype.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vdslib/container/writabledocumentlist.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/select/parser.h> using document::DataType; using document::DocumentTypeRepo; diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp index 9339d23ee86..37a133add91 100644 --- a/documentapi/src/tests/messages/messages52test.cpp +++ b/documentapi/src/tests/messages/messages52test.cpp @@ -5,6 +5,7 @@ #include "messages52test.h" #include <vespa/documentapi/documentapi.h> #include <vespa/document/update/fieldpathupdates.h> +#include <vespa/document/datatype/documenttype.h> using document::DocumentTypeRepo; diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 3527e62840b..5d92b495c48 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -23,7 +23,9 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/document/base/testdocrepo.h> #include <vespa/document/fieldvalue/longfieldvalue.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP("policies_test"); diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp index e3163937a3f..d1ca470f670 100644 --- a/documentapi/src/tests/systemstate/systemstate.cpp +++ b/documentapi/src/tests/systemstate/systemstate.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/documentapi/messagebus/systemstate/systemstate.h> +#include <vespa/documentapi/messagebus/systemstate/nodestate.h> #include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp index 4f0f3508d20..062ed58ca87 100644 --- a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp @@ -3,6 +3,7 @@ #include "loadtypeset.h" #include <vespa/config-load-type.h> #include <vespa/config/config.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/config/helper/configgetter.hpp> diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index 0849a93acbc..6c63e33c745 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -4,16 +4,15 @@ #include "routablefactories51.h" #include "routablefactories52.h" #include "routingpolicyfactories.h" -#include <vespa/document/repo/documenttyperepo.h> +#include "routablerepository.h" +#include "routingpolicyrepository.h" +#include "replymerger.h" #include <vespa/document/util/stringutil.h> #include <vespa/documentapi/documentapi.h> -#include <vespa/documentapi/messagebus/replymerger.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/routing/routingcontext.h> -#include <vespa/vespalib/component/versionspecification.h> #include <vespa/vespalib/util/exceptions.h> -#include "routablerepository.h" -#include "routingpolicyrepository.h" +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".documentprotocol"); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp index 01e95e9f035..aa95cba1785 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp @@ -3,8 +3,7 @@ #include "batchdocumentupdatemessage.h" #include "batchdocumentupdatereply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/document/select/parser.h> -#include <vespa/document/bucket/bucketselector.h> +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/exceptions.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp index e2903097934..af1203926b5 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp @@ -2,6 +2,7 @@ #include "documentmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <cassert> namespace documentapi { @@ -16,7 +17,7 @@ mbus::Reply::UP DocumentMessage::createReply() const { mbus::Reply::UP ret(doCreateReply().release()); - assert(ret.get() != NULL); + assert(ret.get() != nullptr); return ret; } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp index 641feb85f9d..24ac666eae9 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "emptybucketsmessage.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp index e07566cf22f..f23ecdf3556 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "removelocationmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/document/select/parser.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h index c56ef8b0ede..799fbe5bde8 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h @@ -3,8 +3,9 @@ #include "documentmessage.h" #include <vespa/document/bucket/bucketid.h> -#include <vespa/document/bucket/bucketselector.h> -#include <vespa/document/select/parser.h> + +namespace document::select { class Parser; } +namespace document { class BucketIdFactory; } namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 61c6648aee6..4dae20e52d5 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -12,6 +12,7 @@ #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/routing/routingtable.h> #include <vespa/messagebus/messagebus.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".documentrouteselectorpolicy"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp index 3a35e995805..d1a2e7b135b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp @@ -3,9 +3,7 @@ #include <boost/tokenizer.hpp> #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/routing/route.h> -#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/frt/frt.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp index 4d260a8e81f..ba4d0cff079 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp @@ -1,12 +1,26 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "messagetypepolicy.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> #include <vespa/vespalib/stllike/hash_map.hpp> using vespa::config::content::MessagetyperouteselectorpolicyConfig; namespace documentapi { +namespace policy { + +using MessageTypeMapT = vespalib::hash_map<int, mbus::Route>; + +class MessageTypeMap : public MessageTypeMapT { +public: + using MessageTypeMapT::MessageTypeMapT; +}; + +} + MessageTypePolicy::MessageTypePolicy(const config::ConfigUri & configUri) : mbus::IRoutingPolicy(), config::IFetcherCallback<MessagetyperouteselectorpolicyConfig>(), @@ -23,7 +37,7 @@ MessageTypePolicy::~MessageTypePolicy() {} void MessageTypePolicy::configure(std::unique_ptr<MessagetyperouteselectorpolicyConfig> cfg) { - std::unique_ptr<MessageTypeMap> map(new MessageTypeMap); + auto map = std::make_unique<policy::MessageTypeMap>(); for (size_t i(0), m(cfg->route.size()); i < m; i++) { const MessagetyperouteselectorpolicyConfig::Route & r = cfg->route[i]; (*map)[r.messagetype] = mbus::Route::parse(r.name); @@ -38,8 +52,8 @@ void MessageTypePolicy::select(mbus::RoutingContext & context) { int messageType = context.getMessage().getType(); - std::shared_ptr<MessageTypeMap> map = _map.get(); - MessageTypeMap::const_iterator found = map->find(messageType); + std::shared_ptr<policy::MessageTypeMap> map = _map.get(); + policy::MessageTypeMap::const_iterator found = map->find(messageType); if (found != map->end()) { context.addChild(found->second); } else { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h index d9a324a79c5..18ae4d1acb8 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h @@ -1,20 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/select/node.h> -#include <map> #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/messagebus/routing/route.h> -#include <vespa/messagebus/routing/routingcontext.h> -#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/ptrholder.h> #include <vespa/config-messagetyperouteselectorpolicy.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/documentapi/common.h> +namespace mbus { + class RoutingContext; + class Route; +} namespace documentapi { +namespace policy {class MessageTypeMap; } /** * This policy is responsible for selecting among the given recipient routes * according to the configured document selection properties. To factilitate @@ -26,8 +26,7 @@ class MessageTypePolicy : public mbus::IRoutingPolicy, public config::IFetcherCallback<vespa::config::content::MessagetyperouteselectorpolicyConfig> { private: - typedef vespalib::hash_map<int, mbus::Route> MessageTypeMap; - typedef vespalib::PtrHolder<MessageTypeMap> MessageTypeHolder; + typedef vespalib::PtrHolder<policy::MessageTypeMap> MessageTypeHolder; typedef vespalib::PtrHolder<mbus::Route> RouteHolder; MessageTypeHolder _map; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index 5aaf7919d71..f19e0c4c85f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -7,6 +7,7 @@ #include <vespa/documentapi/documentapi.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/config-stor-distribution.h> #include <vespa/config/helper/ifetchercallback.h> #include <vespa/config/helper/configfetcher.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp index 50cf7129016..ef791cc55f1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp @@ -6,6 +6,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> #include <vespa/messagebus/emptyreply.h> +#include <cassert> namespace documentapi { diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index b620611695e..d9c8cd765a1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -4,6 +4,8 @@ #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/select/parser.h> using vespalib::nbostream; using std::make_unique; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp index 4572b2bbcc6..32956ff23bd 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -4,6 +4,8 @@ #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".routablerepository"); @@ -11,9 +13,7 @@ namespace documentapi { RoutableRepository::VersionMap::VersionMap() : _factoryVersions() -{ - // empty -} +{ } bool RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp index 627a760b324..64e05fec793 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "systemstate.h" +#include "nodestate.h" +#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/stringfmt.h> #include <boost/spirit/include/classic_core.hpp> #include <boost/spirit/include/classic_parse_tree.hpp> @@ -256,7 +258,9 @@ parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node< return ret; } -vespalib::Lock SystemState::_parseLock; +namespace { + vespalib::Lock _G_parseLock; +} SystemState::UP SystemState::newInstance(const string &state) @@ -265,7 +269,7 @@ SystemState::newInstance(const string &state) return SystemState::UP(new SystemState(NodeState::UP(new NodeState()))); } try { - vespalib::LockGuard guard(_parseLock); + vespalib::LockGuard guard(_G_parseLock); SystemStateGrammar grammar; boost::spirit::classic::tree_parse_info<> info = boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()), @@ -289,15 +293,14 @@ SystemState::newInstance(const string &state) } } catch(std::exception& e) { - std::cerr << "SystemState::parse() internal error: " - << e.what() << std::endl; + LOG(fatal, "SystemState::parse() internal error: %s", e.what()); } return SystemState::UP(); } SystemState::SystemState(NodeState::UP root) : _root(std::move(root)), - _lock() { - // empty -} + _lock(std::make_unique<vespalib::Lock>()) +{} +SystemState::~SystemState() {}
\ No newline at end of file diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h index 1d958c42b2d..169e1fbb9b6 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "nodestate.h" -#include <vespa/vespalib/util/sync.h> +#include <vespa/documentapi/common.h> +namespace vespalib { class Lock; } namespace documentapi { +class NodeState; + /** * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state * string. The naming of this class is intended to capture the fact that this annotated service tree actually @@ -13,10 +15,8 @@ namespace documentapi { */ class SystemState { private: - static vespalib::Lock _parseLock; - - NodeState::UP _root; - vespalib::Lock _lock; + std::unique_ptr<NodeState> _root; + std::unique_ptr<vespalib::Lock> _lock; friend class SystemStateHandle; @@ -26,9 +26,10 @@ private: * * @param root The root node state. */ - SystemState(NodeState::UP root); + SystemState(std::unique_ptr<NodeState> root); public: + ~SystemState(); SystemState(const SystemState &) = delete; SystemState & operator = (const SystemState &) = delete; /** @@ -47,4 +48,3 @@ public: }; } - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp index 6d3d9a19568..ea43665facb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp @@ -11,14 +11,14 @@ SystemStateHandover::SystemStateHandover(SystemState *state, vespalib::LockGuard SystemStateHandle::SystemStateHandle(SystemState &state) : _state(&state), - _guard(state._lock) + _guard(*state._lock) {} SystemStateHandle::SystemStateHandle(SystemStateHandle &rhs) : _state(rhs._state), _guard(rhs._guard) { - rhs._state = NULL; + rhs._state = nullptr; } SystemStateHandle::SystemStateHandle(const SystemStateHandover &rhs) : @@ -31,7 +31,7 @@ SystemStateHandle::~SystemStateHandle() {} SystemStateHandle::operator SystemStateHandover() { SystemStateHandover ret(_state, _guard); - _state = NULL; + _state = nullptr; return ret; } diff --git a/documentgen-test/OWNERS b/documentgen-test/OWNERS index 123437e2758..e030acdbc5b 100644 --- a/documentgen-test/OWNERS +++ b/documentgen-test/OWNERS @@ -1 +1 @@ -dybis +freva diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index d25995044c2..bccf72e057e 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -69,6 +69,10 @@ search book { field ref type reference<parent> { indexing: attribute } + field vector type tensor(x{}) { + indexing: attribute | summary + attribute: tensor(x{}) + } } field sw1 type float { } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 7e9365a0e6b..b113dc365ba 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -14,6 +14,7 @@ import com.yahoo.document.datatypes.*; import com.yahoo.document.serialization.*; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.tensor.Tensor; import com.yahoo.vespa.document.NodeImpl; import com.yahoo.vespa.document.dom.DocumentImpl; import com.yahoo.vespa.documentgen.test.*; @@ -918,5 +919,13 @@ public class DocumentGenPluginTest { System.out.println(unmasked); assertEquals(unmasked.size(), 0); // probably not needed } + + @Test + public void testTensorType() { + Book book = new Book(new DocumentId("doc:book:0")); + assertNull(book.getVector()); + book.setVector(Tensor.from("{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}")); + assertEquals("{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}", book.getVector().toString()); + } } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 79998163249..d4528cc8d28 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_define_module( APPS src/apps/eval_expr + src/apps/make_tensor_binary_format_test_spec TESTS src/tests/eval/aggr @@ -18,6 +19,7 @@ vespa_define_module( src/tests/eval/param_usage src/tests/eval/simple_tensor src/tests/eval/tensor_function + src/tests/eval/tensor_spec src/tests/eval/value_cache src/tests/eval/value_type src/tests/tensor/dense_dot_product_function diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index d02ef0ad444..804b23db591 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/interpreted_function.h> diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/.gitignore b/eval/src/apps/make_tensor_binary_format_test_spec/.gitignore new file mode 100644 index 00000000000..aff1914d6bf --- /dev/null +++ b/eval/src/apps/make_tensor_binary_format_test_spec/.gitignore @@ -0,0 +1 @@ +/eval_make_tensor_binary_format_test_spec_app diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/CMakeLists.txt b/eval/src/apps/make_tensor_binary_format_test_spec/CMakeLists.txt new file mode 100644 index 00000000000..b12bd919124 --- /dev/null +++ b/eval/src/apps/make_tensor_binary_format_test_spec/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_make_tensor_binary_format_test_spec_app + SOURCES + make_tensor_binary_format_test_spec.cpp + DEPENDS + vespaeval +) diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp new file mode 100644 index 00000000000..2967455236e --- /dev/null +++ b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp @@ -0,0 +1,320 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_type.h> +#include <iostream> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::slime::convenience; + +using Options = std::initializer_list<std::reference_wrapper<const nbostream>>; +using Dict = std::vector<vespalib::string>; + +//----------------------------------------------------------------------------- + +nbostream make_sparse() { + nbostream data; + data << uint8_t(0x1); + return data; +} + +nbostream make_dense() { + nbostream data; + data << uint8_t(0x2); + return data; +} + +nbostream make_mixed() { + nbostream data; + data << uint8_t(0x3); + return data; +} + +void set_tensor(Cursor &test, const TensorSpec &spec) { + const Inspector &old_tensor = test["tensor"]; + if (old_tensor.valid()) { + TensorSpec old_spec = TensorSpec::from_slime(old_tensor); + if (!(old_spec == spec)) { + fprintf(stderr, "inconsistent specs\n"); + std::cerr << old_spec; + std::cerr << spec; + abort(); // inconsistent specs across binary permutations + } + } else { + Cursor &tensor = test.setObject("tensor"); + spec.to_slime(tensor); + } +} + +void add_binary(Cursor &test, const nbostream &data) { + if (!test["binary"].valid()) { + test.setArray("binary"); + } + test["binary"].addData(Memory(data.peek(), data.size())); +} + +void add_binary(Cursor &test, Options opts) { + for (const nbostream &opt: opts) { + add_binary(test, opt); + } +} + +std::vector<Dict> make_permutations(const Dict &dict) { + std::vector<Dict> list; + if (dict.empty()) { + } else if (dict.size() == 1) { + list.push_back(dict); + } else if (dict.size() == 2) { + list.push_back({dict[0], dict[1]}); + list.push_back({dict[1], dict[0]}); + } else if (dict.size() == 3) { + list.push_back({dict[0], dict[1], dict[2]}); + list.push_back({dict[0], dict[2], dict[1]}); + list.push_back({dict[1], dict[0], dict[2]}); + list.push_back({dict[1], dict[2], dict[0]}); + list.push_back({dict[2], dict[0], dict[1]}); + list.push_back({dict[2], dict[1], dict[0]}); + } else { + fprintf(stderr, "unsupported permutation size: %zu\n", dict.size()); + abort(); // only implemented for sizes (0,1,2,3) + } + return list; +}; + +const std::map<std::string, double> val_map{ + {"a", 1.0}, + {"b", 2.0}, + {"c", 3.0}, + {"foo", 1.0}, + {"bar", 2.0}}; + +double val(size_t idx) { return double(idx + 1); } +double val(const vespalib::string &label) { + auto res = val_map.find(label); + if (res == val_map.end()) { + fprintf(stderr, "unsupported label: '%s'\n", label.c_str()); + abort(); // unsupported label + } + return res->second; +} +double mix(std::initializer_list<double> vals) { + double value = 0.0; + for (double val: vals) { + value = ((value * 10) + val); + } + return value; +} + +//----------------------------------------------------------------------------- + +void make_number_test(Cursor &test, double value) { + TensorSpec spec("double"); + spec.add({{}}, value); + nbostream sparse = make_sparse(); + sparse.putInt1_4Bytes(0); + sparse.putInt1_4Bytes(1); + sparse << value; + nbostream dense = make_dense(); + dense.putInt1_4Bytes(0); + dense << value; + nbostream mixed = make_mixed(); + mixed.putInt1_4Bytes(0); + mixed.putInt1_4Bytes(0); + mixed << value; + set_tensor(test, spec); + add_binary(test, {sparse, dense, mixed}); + if (value == 0.0) { + nbostream empty = make_sparse(); + empty.putInt1_4Bytes(0); + empty.putInt1_4Bytes(0); + add_binary(test, empty); + } +} + +//----------------------------------------------------------------------------- + +void make_vector_test(Cursor &test, size_t x_size) { + TensorSpec spec(vespalib::make_string("tensor(x[%zu])", x_size)); + nbostream dense = make_dense(); + dense.putInt1_4Bytes(1); + dense.writeSmallString("x"); + dense.putInt1_4Bytes(x_size); + nbostream mixed = make_mixed(); + mixed.putInt1_4Bytes(0); + mixed.putInt1_4Bytes(1); + mixed.writeSmallString("x"); + mixed.putInt1_4Bytes(x_size); + for (size_t x = 0; x < x_size; ++x) { + double value = val(x); + spec.add({{"x", x}}, value); + dense << value; + mixed << value; + } + set_tensor(test, spec); + add_binary(test, {dense, mixed}); +} + +void make_matrix_test(Cursor &test, size_t x_size, size_t y_size) { + TensorSpec spec(vespalib::make_string("tensor(x[%zu],y[%zu])", x_size, y_size)); + nbostream dense = make_dense(); + dense.putInt1_4Bytes(2); + dense.writeSmallString("x"); + dense.putInt1_4Bytes(x_size); + dense.writeSmallString("y"); + dense.putInt1_4Bytes(y_size); + nbostream mixed = make_mixed(); + mixed.putInt1_4Bytes(0); + mixed.putInt1_4Bytes(2); + mixed.writeSmallString("x"); + mixed.putInt1_4Bytes(x_size); + mixed.writeSmallString("y"); + mixed.putInt1_4Bytes(y_size); + for (size_t x = 0; x < x_size; ++x) { + for (size_t y = 0; y < y_size; ++y) { + double value = mix({val(x), val(y)}); + spec.add({{"x", x}, {"y", y}}, value); + dense << value; + mixed << value; + } + } + set_tensor(test, spec); + add_binary(test, {dense, mixed}); +} + +//----------------------------------------------------------------------------- + +void make_map_test(Cursor &test, const Dict &x_dict_in) { + TensorSpec spec("tensor(x{})"); + nbostream sparse_base = make_sparse(); + sparse_base.putInt1_4Bytes(1); + sparse_base.writeSmallString("x"); + sparse_base.putInt1_4Bytes(x_dict_in.size()); + nbostream mixed_base = make_mixed(); + mixed_base.putInt1_4Bytes(1); + mixed_base.writeSmallString("x"); + mixed_base.putInt1_4Bytes(0); + mixed_base.putInt1_4Bytes(x_dict_in.size()); + auto x_perm = make_permutations(x_dict_in); + for (const Dict &x_dict: x_perm) { + nbostream sparse = sparse_base; + nbostream mixed = mixed_base; + for (vespalib::string x: x_dict) { + double value = val(x); + spec.add({{"x", x}}, value); + sparse.writeSmallString(x); + mixed.writeSmallString(x); + sparse << value; + mixed << value; + } + set_tensor(test, spec); + add_binary(test, {sparse, mixed}); + } + if (x_dict_in.empty()) { + set_tensor(test, spec); + add_binary(test, {sparse_base, mixed_base}); + } +} + +void make_mesh_test(Cursor &test, const Dict &x_dict_in, const vespalib::string &y) { + TensorSpec spec("tensor(x{},y{})"); + nbostream sparse_base = make_sparse(); + sparse_base.putInt1_4Bytes(2); + sparse_base.writeSmallString("x"); + sparse_base.writeSmallString("y"); + sparse_base.putInt1_4Bytes(x_dict_in.size() * 1); + nbostream mixed_base = make_mixed(); + mixed_base.putInt1_4Bytes(2); + mixed_base.writeSmallString("x"); + mixed_base.writeSmallString("y"); + mixed_base.putInt1_4Bytes(0); + mixed_base.putInt1_4Bytes(x_dict_in.size() * 1); + auto x_perm = make_permutations(x_dict_in); + for (const Dict &x_dict: x_perm) { + nbostream sparse = sparse_base; + nbostream mixed = mixed_base; + for (vespalib::string x: x_dict) { + double value = mix({val(x), val(y)}); + spec.add({{"x", x}, {"y", y}}, value); + sparse.writeSmallString(x); + sparse.writeSmallString(y); + mixed.writeSmallString(x); + mixed.writeSmallString(y); + sparse << value; + mixed << value; + } + set_tensor(test, spec); + add_binary(test, {sparse, mixed}); + } + if (x_dict_in.empty()) { + set_tensor(test, spec); + add_binary(test, {sparse_base, mixed_base}); + } +} + +//----------------------------------------------------------------------------- + +void make_vector_map_test(Cursor &test, + const vespalib::string &mapped_name, const Dict &mapped_dict, + const vespalib::string &indexed_name, size_t indexed_size) +{ + auto type_str = vespalib::make_string("tensor(%s{},%s[%zu])", + mapped_name.c_str(), indexed_name.c_str(), indexed_size); + ValueType type = ValueType::from_spec(type_str); + TensorSpec spec(type.to_spec()); // ensures type string is normalized + nbostream mixed_base = make_mixed(); + mixed_base.putInt1_4Bytes(1); + mixed_base.writeSmallString(mapped_name); + mixed_base.putInt1_4Bytes(1); + mixed_base.writeSmallString(indexed_name); + mixed_base.putInt1_4Bytes(indexed_size); + mixed_base.putInt1_4Bytes(mapped_dict.size()); + auto mapped_perm = make_permutations(mapped_dict); + for (const Dict &dict: mapped_perm) { + nbostream mixed = mixed_base; + for (vespalib::string label: dict) { + mixed.writeSmallString(label); + for (size_t idx = 0; idx < indexed_size; ++idx) { + double value = mix({val(label), val(idx)}); + spec.add({{mapped_name, label}, {indexed_name, idx}}, value); + mixed << value; + } + } + set_tensor(test, spec); + add_binary(test, mixed); + } + if (mapped_dict.empty()) { + set_tensor(test, spec); + add_binary(test, mixed_base); + } +} + +//----------------------------------------------------------------------------- + +void make_tests(Cursor &tests) { + make_number_test(tests.addObject(), 0.0); + make_number_test(tests.addObject(), 42.0); + make_vector_test(tests.addObject(), 3); + make_matrix_test(tests.addObject(), 2, 3); + make_map_test(tests.addObject(), {}); + make_map_test(tests.addObject(), {"a", "b", "c"}); + make_mesh_test(tests.addObject(), {}, "a"); + make_mesh_test(tests.addObject(), {"foo", "bar"}, "a"); + make_vector_map_test(tests.addObject(), "x", {}, "y", 10); + make_vector_map_test(tests.addObject(), "y", {}, "x", 10); + make_vector_map_test(tests.addObject(), "x", {"a", "b"}, "y", 3); + make_vector_map_test(tests.addObject(), "y", {"a", "b"}, "x", 3); +} + +int main(int, char **) { + Slime slime; + Cursor &top = slime.setObject(); + Cursor &tests = top.setArray("tests"); + make_tests(tests); + top.setLong("num_tests", tests.entries()); + fprintf(stdout, "%s", slime.toString().c_str()); + return 0; +} diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json b/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json new file mode 100644 index 00000000000..1b74b4b8838 --- /dev/null +++ b/eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json @@ -0,0 +1,336 @@ +{ + "tests": [ + { + "tensor": { + "type": "double", + "cells": [ + { + "address": { + }, + "value": 0 + } + ] + }, + "binary": [ + "0x0100010000000000000000", + "0x02000000000000000000", + "0x0300000000000000000000", + "0x010000" + ] + }, + { + "tensor": { + "type": "double", + "cells": [ + { + "address": { + }, + "value": 42 + } + ] + }, + "binary": [ + "0x0100014045000000000000", + "0x02004045000000000000", + "0x0300004045000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x[3])", + "cells": [ + { + "address": { + "x": 0 + }, + "value": 1 + }, + { + "address": { + "x": 1 + }, + "value": 2 + }, + { + "address": { + "x": 2 + }, + "value": 3 + } + ] + }, + "binary": [ + "0x02010178033FF000000000000040000000000000004008000000000000", + "0x0300010178033FF000000000000040000000000000004008000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x[2],y[3])", + "cells": [ + { + "address": { + "x": 0, + "y": 0 + }, + "value": 11 + }, + { + "address": { + "x": 0, + "y": 1 + }, + "value": 12 + }, + { + "address": { + "x": 0, + "y": 2 + }, + "value": 13 + }, + { + "address": { + "x": 1, + "y": 0 + }, + "value": 21 + }, + { + "address": { + "x": 1, + "y": 1 + }, + "value": 22 + }, + { + "address": { + "x": 1, + "y": 2 + }, + "value": 23 + } + ] + }, + "binary": [ + "0x020201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000", + "0x03000201780201790340260000000000004028000000000000402A000000000000403500000000000040360000000000004037000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x{})", + "cells": [ + ] + }, + "binary": [ + "0x0101017800", + "0x030101780000" + ] + }, + { + "tensor": { + "type": "tensor(x{})", + "cells": [ + { + "address": { + "x": "a" + }, + "value": 1 + }, + { + "address": { + "x": "b" + }, + "value": 2 + }, + { + "address": { + "x": "c" + }, + "value": 3 + } + ] + }, + "binary": [ + "0x010101780301613FF00000000000000162400000000000000001634008000000000000", + "0x03010178000301613FF00000000000000162400000000000000001634008000000000000", + "0x010101780301613FF00000000000000163400800000000000001624000000000000000", + "0x03010178000301613FF00000000000000163400800000000000001624000000000000000", + "0x01010178030162400000000000000001613FF000000000000001634008000000000000", + "0x0301017800030162400000000000000001613FF000000000000001634008000000000000", + "0x0101017803016240000000000000000163400800000000000001613FF0000000000000", + "0x030101780003016240000000000000000163400800000000000001613FF0000000000000", + "0x01010178030163400800000000000001613FF000000000000001624000000000000000", + "0x0301017800030163400800000000000001613FF000000000000001624000000000000000", + "0x0101017803016340080000000000000162400000000000000001613FF0000000000000", + "0x030101780003016340080000000000000162400000000000000001613FF0000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x{},y{})", + "cells": [ + ] + }, + "binary": [ + "0x01020178017900", + "0x0302017801790000" + ] + }, + { + "tensor": { + "type": "tensor(x{},y{})", + "cells": [ + { + "address": { + "x": "bar", + "y": "a" + }, + "value": 21 + }, + { + "address": { + "x": "foo", + "y": "a" + }, + "value": 11 + } + ] + }, + "binary": [ + "0x0102017801790203666F6F016140260000000000000362617201614035000000000000", + "0x030201780179000203666F6F016140260000000000000362617201614035000000000000", + "0x01020178017902036261720161403500000000000003666F6F01614026000000000000", + "0x0302017801790002036261720161403500000000000003666F6F01614026000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x{},y[10])", + "cells": [ + ] + }, + "binary": [ + "0x030101780101790A00" + ] + }, + { + "tensor": { + "type": "tensor(x[10],y{})", + "cells": [ + ] + }, + "binary": [ + "0x030101790101780A00" + ] + }, + { + "tensor": { + "type": "tensor(x{},y[3])", + "cells": [ + { + "address": { + "x": "a", + "y": 0 + }, + "value": 11 + }, + { + "address": { + "x": "a", + "y": 1 + }, + "value": 12 + }, + { + "address": { + "x": "a", + "y": 2 + }, + "value": 13 + }, + { + "address": { + "x": "b", + "y": 0 + }, + "value": 21 + }, + { + "address": { + "x": "b", + "y": 1 + }, + "value": 22 + }, + { + "address": { + "x": "b", + "y": 2 + }, + "value": 23 + } + ] + }, + "binary": [ + "0x030101780101790302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000", + "0x0301017801017903020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000" + ] + }, + { + "tensor": { + "type": "tensor(x[3],y{})", + "cells": [ + { + "address": { + "x": 0, + "y": "a" + }, + "value": 11 + }, + { + "address": { + "x": 0, + "y": "b" + }, + "value": 21 + }, + { + "address": { + "x": 1, + "y": "a" + }, + "value": 12 + }, + { + "address": { + "x": 1, + "y": "b" + }, + "value": 22 + }, + { + "address": { + "x": 2, + "y": "a" + }, + "value": 13 + }, + { + "address": { + "x": 2, + "y": "b" + }, + "value": 23 + } + ] + }, + "binary": [ + "0x030101790101780302016140260000000000004028000000000000402A0000000000000162403500000000000040360000000000004037000000000000", + "0x0301017901017803020162403500000000000040360000000000004037000000000000016140260000000000004028000000000000402A000000000000" + ] + } + ], + "num_tests": 12 +} diff --git a/eval/src/tests/eval/gbdt/gbdt_test.cpp b/eval/src/tests/eval/gbdt/gbdt_test.cpp index c96779edeb0..0535c7280bf 100644 --- a/eval/src/tests/eval/gbdt/gbdt_test.cpp +++ b/eval/src/tests/eval/gbdt/gbdt_test.cpp @@ -323,4 +323,34 @@ TEST("require that forests evaluate to approximately the same for all evaluation //----------------------------------------------------------------------------- +TEST("require that GDBT expressions can be detected") { + Function function = Function::parse("if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0))"); + EXPECT_TRUE(contains_gbdt(function.root(), 9)); + EXPECT_TRUE(!contains_gbdt(function.root(), 10)); +} + +TEST("require that wrapped GDBT expressions can be detected") { + Function function = Function::parse("10*(if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0)))"); + EXPECT_TRUE(contains_gbdt(function.root(), 9)); + EXPECT_TRUE(!contains_gbdt(function.root(), 10)); +} + +TEST("require that lazy parameters are not suggested for GBDT models") { + Function function = Function::parse(Model().make_forest(10, 8)); + EXPECT_TRUE(!CompiledFunction::should_use_lazy_params(function)); +} + +TEST("require that lazy parameters can be suggested for small GBDT models") { + Function function = Function::parse("if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0))+" + "if((d in 1),10.0,if((e<1),20.0,30.0))"); + EXPECT_TRUE(CompiledFunction::should_use_lazy_params(function)); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/param_usage/param_usage_test.cpp b/eval/src/tests/eval/param_usage/param_usage_test.cpp index ff0c6667279..24478046289 100644 --- a/eval/src/tests/eval/param_usage/param_usage_test.cpp +++ b/eval/src/tests/eval/param_usage/param_usage_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/param_usage.h> +#include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/vespalib/test/insertion_operators.h> using vespalib::approx_equal; @@ -62,4 +63,14 @@ TEST("require that multi-level if statements are combined correctly") { EXPECT_EQUAL(List(check_param_usage(function)), List({1.0, 0.5, 1.0, 1.0})); } +TEST("require that lazy parameters are suggested for functions with parameters that might not be used") { + Function function = Function::parse("if(z,x,y)+if(w,y,x)"); + EXPECT_TRUE(CompiledFunction::should_use_lazy_params(function)); +} + +TEST("require that lazy parameters are not suggested for functions where all parameters are always used") { + Function function = Function::parse("a*b*c"); + EXPECT_TRUE(!CompiledFunction::should_use_lazy_params(function)); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/tensor_spec/CMakeLists.txt b/eval/src/tests/eval/tensor_spec/CMakeLists.txt new file mode 100644 index 00000000000..65e927f8b61 --- /dev/null +++ b/eval/src/tests/eval/tensor_spec/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_tensor_spec_test_app TEST + SOURCES + tensor_spec_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_tensor_spec_test_app COMMAND eval_tensor_spec_test_app) diff --git a/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp b/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp new file mode 100644 index 00000000000..ecbaf292037 --- /dev/null +++ b/eval/src/tests/eval/tensor_spec/tensor_spec_test.cpp @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/vespalib/data/slime/slime.h> + +using vespalib::Slime; +using vespalib::eval::TensorSpec; + +TEST("require that a tensor spec can be converted to and from slime") { + TensorSpec spec("tensor(x[2],y{})"); + spec.add({{"x", 0}, {"y", "xxx"}}, 1.0) + .add({{"x", 0}, {"y", "yyy"}}, 2.0) + .add({{"x", 1}, {"y", "xxx"}}, 3.0) + .add({{"x", 1}, {"y", "yyy"}}, 4.0); + Slime slime; + spec.to_slime(slime.setObject()); + fprintf(stderr, "tensor spec as slime: \n%s\n", slime.get().toString().c_str()); + EXPECT_EQUAL(TensorSpec::from_slime(slime.get()), spec); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index 1865f863e4d..726bd4be3cf 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -8,12 +8,15 @@ using vespalib::eval::SimpleTensorEngine; using vespalib::eval::test::TensorConformance; using vespalib::tensor::DefaultTensorEngine; +vespalib::string module_path(TEST_PATH("../../../../")); + + TEST("require that reference tensor implementation passes all conformance tests") { - TEST_DO(TensorConformance::run_tests(SimpleTensorEngine::ref(), true)); + TEST_DO(TensorConformance::run_tests(module_path, SimpleTensorEngine::ref())); } -IGNORE_TEST("require that production tensor implementation passes non-mixed conformance tests") { - TEST_DO(TensorConformance::run_tests(DefaultTensorEngine::ref(), false)); +TEST("require that production tensor implementation passes all conformance tests") { + TEST_DO(TensorConformance::run_tests(module_path, DefaultTensorEngine::ref())); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp index 8fede7e3d0b..6572ab1ed92 100644 --- a/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp +++ b/eval/src/tests/tensor/tensor_mapper/tensor_mapper_test.cpp @@ -2,244 +2,235 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_builder.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor_builder.h> -#include <vespa/eval/tensor/types.h> -#include <vespa/eval/tensor/tensor_factory.h> #include <vespa/eval/tensor/tensor_mapper.h> -#include <vespa/eval/tensor/default_tensor.h> -#include <ostream> +#include <vespa/eval/tensor/wrapped_simple_tensor.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/simple_tensor.h> using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; +using vespalib::eval::SimpleTensor; using namespace vespalib::tensor; -namespace vespalib { -namespace tensor { +void verify_wrapped(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) { + auto tensor = std::make_unique<WrappedSimpleTensor>(SimpleTensor::create(source)); + auto mapped = TensorMapper::mapToWrapped(*tensor, ValueType::from_spec(type)); + TensorSpec actual = mapped->toSpec(); + EXPECT_EQUAL(actual, expect); +} + +void verify(const TensorSpec &source, const vespalib::string &type, const TensorSpec &expect) { + auto tensor = DefaultTensorEngine::ref().create(source); + const Tensor *tensor_impl = dynamic_cast<const Tensor *>(tensor.get()); + ASSERT_TRUE(tensor_impl); + TensorMapper mapper(ValueType::from_spec(type)); + auto mapped = mapper.map(*tensor_impl); + TensorSpec actual = mapped->toSpec(); + EXPECT_EQUAL(actual, expect); + TEST_DO(verify_wrapped(source, type, expect)); +} + +TEST("require that sparse tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","1"}}, 1) + .add({{"x","2"},{"y","1"}}, 3) + .add({{"x","1"},{"y","2"}}, 5) + .add({{"x","2"},{"y","2"}}, 7), + "tensor(y{})", + TensorSpec("tensor(y{})") + .add({{"y","1"}}, 4) + .add({{"y","2"}}, 12))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","1"}}, 1) + .add({{"x","2"},{"y","1"}}, 3) + .add({{"x","1"},{"y","2"}}, 5) + .add({{"x","2"},{"y","2"}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","1"}}, 6) + .add({{"x","2"}}, 10))); +} + +TEST("require that sparse tensors can be mapped to dense type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","0"}}, 1) + .add({{"x","2"},{"y","0"}}, 3) + .add({{"x","1"},{"y","1"}}, 5) + .add({{"x","2"},{"y","1"}}, 7), + "tensor(y[3])", + TensorSpec("tensor(y[3])") + .add({{"y",0}}, 4) + .add({{"y",1}}, 12) + .add({{"y",2}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","1"},{"y","0x"}}, 1) + .add({{"x","2"},{"y",""}}, 3) + .add({{"x","1"},{"y","1"}}, 5) + .add({{"x","2"},{"y","10"}}, 7), + "tensor(y[3])", + TensorSpec("tensor(y[3])") + .add({{"y",0}}, 3) + .add({{"y",1}}, 5) + .add({{"y",2}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","1"}}, 7), + "tensor(x[2],y[3])", + TensorSpec("tensor(x[2],y[3])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",0},{"y",2}}, 0) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",1},{"y",2}}, 0))); +} -static bool operator==(const Tensor &lhs, const Tensor &rhs) -{ - return lhs.equals(rhs); +TEST("require that sparse tensors can be mapped to abstract dense type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","1"}}, 7), + "tensor(x[2],y[])", + TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","2"},{"y","0"}}, 7), + "tensor(x[],y[])", + TensorSpec("tensor(x[3],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",2},{"y",0}}, 7) + .add({{"x",2},{"y",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","1"},{"y","0"}}, 3) + .add({{"x","0"},{"y","1"}}, 5) + .add({{"x","10"},{"y","3"}}, 7), + "tensor(x[],y[3])", + TensorSpec("tensor(x[2],y[3])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 5) + .add({{"x",0},{"y",2}}, 0) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 0) + .add({{"x",1},{"y",2}}, 0))); } +TEST("require that dense tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 3) + .add({{"x",1},{"y",0}}, 5) + .add({{"x",1},{"y",1}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","0"}}, 4) + .add({{"x","1"}}, 12))); } + +TEST("require that mixed tensors can be mapped to sparse type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(x{})", + TensorSpec("tensor(x{})") + .add({{"x","0"}}, 4) + .add({{"x","1"}}, 12))); +} + +TEST("require that mixed tensors can be mapped to dense type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(y[])", + TensorSpec("tensor(y[2])") + .add({{"y",0}}, 6) + .add({{"y",1}}, 10))); +} + +TEST("require that mixed tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7), + "tensor(x{},y[])", + TensorSpec("tensor(x{},y[2])") + .add({{"x","0"},{"y",0}}, 1) + .add({{"x","0"},{"y",1}}, 3) + .add({{"x","1"},{"y",0}}, 5) + .add({{"x","1"},{"y",1}}, 7))); +} + +TEST("require that dense tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x[2],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 3) + .add({{"x",1},{"y",0}}, 5) + .add({{"x",1},{"y",1}}, 7), + "tensor(x{},y[])", + TensorSpec("tensor(x{},y[2])") + .add({{"x","0"},{"y",0}}, 1) + .add({{"x","0"},{"y",1}}, 3) + .add({{"x","1"},{"y",0}}, 5) + .add({{"x","1"},{"y",1}}, 7))); } -template <typename BuilderType> -bool defaultBuilder() { return false; } - -template <> -bool defaultBuilder<DefaultTensor::builder>() { return true; } - -template <typename BuilderType> -struct TensorTFromBuilder; - -template <> -struct TensorTFromBuilder<SparseTensorBuilder> { - using TensorT = SparseTensor; -}; - -template <typename BuilderType> -using TensorTFromBuilder_t = typename TensorTFromBuilder<BuilderType>::TensorT; - -struct FixtureBase -{ - Tensor::UP createDenseTensor(const DenseTensorCells &cells) { - return TensorFactory::createDense(cells); - } -}; - -template <typename BuilderType> -struct Fixture : public FixtureBase -{ - BuilderType _builder; - using TensorT = TensorTFromBuilder_t<BuilderType>; - Fixture() : FixtureBase(), _builder() {} - - Tensor::UP createTensor(const TensorCells &cells, - const TensorDimensions &dimensions) { - return TensorFactory::create(cells, dimensions, _builder); - } - - void assertSparseMapImpl(const Tensor &exp, - const ValueType &tensorType, - const Tensor &rhs, bool isDefaultBuilder) - { - EXPECT_TRUE(tensorType.is_sparse()); - if (isDefaultBuilder) { - TensorMapper mapper(tensorType); - std::unique_ptr<Tensor> mapped = mapper.map(rhs); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - std::unique_ptr<Tensor> mapped = - TensorMapper::mapToSparse<TensorT>(rhs, tensorType); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - - void assertDenseMapImpl(const Tensor &exp, - const ValueType &tensorType, - const Tensor &rhs) - { - EXPECT_TRUE(tensorType.is_dense()); - TensorMapper mapper(tensorType); - std::unique_ptr<Tensor> mapped = mapper.map(rhs); - EXPECT_TRUE(!!mapped); - EXPECT_EQUAL(exp, *mapped); - } - - void - assertSparseMap(const TensorCells &expTensor, - const TensorDimensions &expDimensions, - const vespalib::string &typeSpec, - const TensorCells &rhsTensor, - const TensorDimensions &rhsDimensions) - { - assertSparseMapImpl(*createTensor(expTensor, expDimensions), - ValueType::from_spec(typeSpec), - *createTensor(rhsTensor, rhsDimensions), - defaultBuilder<BuilderType>()); - } - - void - assertDenseMap(const DenseTensorCells &expTensor, - const vespalib::string &typeSpec, - const TensorCells &rhsTensor, - const TensorDimensions &rhsDimensions) - { - assertDenseMapImpl(*createDenseTensor(expTensor), - ValueType::from_spec(typeSpec), - *createTensor(rhsTensor, rhsDimensions)); - } -}; - -using SparseFixture = Fixture<SparseTensorBuilder>; - -template <typename FixtureType> -void -testTensorMapper(FixtureType &f) -{ - TEST_DO(f.assertSparseMap({ - {{{"y","1"}}, 4}, - {{{"y","2"}}, 12} - }, - { "y" }, - "tensor(y{})", - { - {{{"x","1"},{"y","1"}}, 1}, - {{{"x","2"},{"y","1"}}, 3}, - {{{"x","1"},{"y","2"}}, 5}, - {{{"x","2"},{"y","2"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertSparseMap({ - {{{"x","1"}}, 6}, - {{{"x","2"}}, 10} - }, - { "x" }, - "tensor(x{})", - { - {{{"x","1"},{"y","1"}}, 1}, - {{{"x","2"},{"y","1"}}, 3}, - {{{"x","1"},{"y","2"}}, 5}, - {{{"x","2"},{"y","2"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"y",0}}, 4}, - {{{"y",1}}, 12}, - {{{"y",2}}, 0} - }, - "tensor(y[3])", - { - {{{"x","1"},{"y","0"}}, 1}, - {{{"x","2"},{"y","0"}}, 3}, - {{{"x","1"},{"y","1"}}, 5}, - {{{"x","2"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"y",0}}, 3}, - {{{"y",1}}, 5}, - {{{"y",2}}, 0} - }, - "tensor(y[3])", - { - {{{"x","1"},{"y","0x"}}, 1}, - {{{"x","2"},{"y",""}}, 3}, - {{{"x","1"},{"y","1"}}, 5}, - {{{"x","2"},{"y","10"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",0},{"y",2}}, 0}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",1},{"y",2}}, 0} - }, - "tensor(x[2], y[3])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0} - }, - "tensor(x[2], y[])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","1"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",2},{"y",0}}, 7}, - {{{"x",2},{"y",1}}, 0} - }, - "tensor(x[], y[])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","2"},{"y","0"}}, 7} - }, - { "x", "y" })); - TEST_DO(f.assertDenseMap({ - {{{"x",0},{"y",0}}, 1}, - {{{"x",0},{"y",1}}, 5}, - {{{"x",0},{"y",2}}, 0}, - {{{"x",1},{"y",0}}, 3}, - {{{"x",1},{"y",1}}, 0}, - {{{"x",1},{"y",2}}, 0} - }, - "tensor(x[], y[3])", - { - {{{"x","0"},{"y","0"}}, 1}, - {{{"x","1"},{"y","0"}}, 3}, - {{{"x","0"},{"y","1"}}, 5}, - {{{"x","10"},{"y","3"}}, 7} - }, - { "x", "y" })); +TEST("require that sparse tensors can be mapped to mixed type") { + TEST_DO(verify(TensorSpec("tensor(x{},y{})") + .add({{"x","0"},{"y","0"}}, 1) + .add({{"x","0"},{"y","1"}}, 3) + .add({{"x","1"},{"y","0"}}, 5) + .add({{"x","1"},{"y","1"}}, 7), + "tensor(x[],y{})", + TensorSpec("tensor(x[2],y{})") + .add({{"x",0},{"y","0"}}, 1) + .add({{"x",0},{"y","1"}}, 3) + .add({{"x",1},{"y","0"}}, 5) + .add({{"x",1},{"y","1"}}, 7))); } -TEST_F("test tensor mapper for SparseTensor", SparseFixture) -{ - testTensorMapper(f); +TEST("require that missing dimensions are added appropriately") { + TEST_DO(verify(TensorSpec("tensor(x{})") + .add({{"x","foo"}}, 42), + "tensor(x{},y{})", + TensorSpec("tensor(x{},y{})") + .add({{"x","foo"},{"y",""}}, 42))); + + TEST_DO(verify(TensorSpec("tensor(x[1])") + .add({{"x",0}}, 42), + "tensor(x[1],y[],z[2])", + TensorSpec("tensor(x[1],y[1],z[2])") + .add({{"x",0},{"y",0},{"z",0}}, 42) + .add({{"x",0},{"y",0},{"z",1}}, 0))); + + TEST_DO(verify(TensorSpec("tensor(a{})") + .add({{"a","foo"}}, 42), + "tensor(a{},b[],c{},d[2])", + TensorSpec("tensor(a{},b[1],c{},d[2])") + .add({{"a","foo"},{"b",0},{"c",""},{"d",0}}, 42) + .add({{"a","foo"},{"b",0},{"c",""},{"d",1}}, 0))); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/aggr.cpp b/eval/src/vespa/eval/eval/aggr.cpp index 51d20403488..a69d5252c2f 100644 --- a/eval/src/vespa/eval/eval/aggr.cpp +++ b/eval/src/vespa/eval/eval/aggr.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/stash.h> #include "aggr.h" +#include <vespa/vespalib/util/stash.h> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/basic_nodes.cpp b/eval/src/vespa/eval/eval/basic_nodes.cpp index 41ecab014d0..2e5feb16698 100644 --- a/eval/src/vespa/eval/eval/basic_nodes.cpp +++ b/eval/src/vespa/eval/eval/basic_nodes.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "basic_nodes.h" #include "node_traverser.h" #include "node_visitor.h" diff --git a/eval/src/vespa/eval/eval/delete_node.cpp b/eval/src/vespa/eval/eval/delete_node.cpp index 61204c890b2..2ab8ab676f3 100644 --- a/eval/src/vespa/eval/eval/delete_node.cpp +++ b/eval/src/vespa/eval/eval/delete_node.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "key_gen.h" #include "node_visitor.h" #include "node_traverser.h" diff --git a/eval/src/vespa/eval/eval/function.cpp b/eval/src/vespa/eval/eval/function.cpp index dd8d7743041..82de221a065 100644 --- a/eval/src/vespa/eval/eval/function.cpp +++ b/eval/src/vespa/eval/eval/function.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <cctype> -#include <map> #include "function.h" #include "basic_nodes.h" #include "tensor_nodes.h" @@ -10,6 +7,8 @@ #include "call_nodes.h" #include "delete_node.h" #include "aggr.h" +#include <cctype> +#include <map> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/gbdt.cpp b/eval/src/vespa/eval/eval/gbdt.cpp index 26368b1e2e4..b787a9a04f0 100644 --- a/eval/src/vespa/eval/eval/gbdt.cpp +++ b/eval/src/vespa/eval/eval/gbdt.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "gbdt.h" +#include "vm_forest.h" +#include "node_traverser.h" #include <vespa/eval/eval/basic_nodes.h> #include <vespa/eval/eval/call_nodes.h> #include <vespa/eval/eval/operator_nodes.h> -#include "vm_forest.h" namespace vespalib { namespace eval { @@ -118,6 +118,25 @@ ForestStats::ForestStats(const std::vector<const nodes::Node *> &trees) //----------------------------------------------------------------------------- +bool contains_gbdt(const nodes::Node &node, size_t limit) { + struct FindGBDT : NodeTraverser { + size_t seen; + size_t limit; + explicit FindGBDT(size_t limit_in) : seen(0), limit(limit_in) {} + bool found() const { return (seen >= limit); } + bool open(const nodes::Node &) override { return !found(); } + void close(const nodes::Node &node) override { + if (node.is_tree() || node.is_forest()) { + ++seen; + } + } + } findGBDT(limit); + node.traverse(findGBDT); + return findGBDT.found(); +} + +//----------------------------------------------------------------------------- + Optimize::Result Optimize::select_best(const ForestStats &stats, const std::vector<const nodes::Node *> &trees) diff --git a/eval/src/vespa/eval/eval/gbdt.h b/eval/src/vespa/eval/eval/gbdt.h index 1c22c195d46..eda0d16229f 100644 --- a/eval/src/vespa/eval/eval/gbdt.h +++ b/eval/src/vespa/eval/eval/gbdt.h @@ -3,6 +3,7 @@ #pragma once #include <vector> +#include <memory> namespace vespalib { namespace eval { @@ -59,6 +60,15 @@ struct ForestStats { //----------------------------------------------------------------------------- /** + * Check if the given sub-expression contains GBDT. This function + * returns true if the number of tree/forest nodes exceeds the given + * limit. + **/ +bool contains_gbdt(const nodes::Node &node, size_t limit); + +//----------------------------------------------------------------------------- + +/** * A Forest object represents deletable custom prepared state that may * be used to evaluate a GBDT forest from within LLVM generated * machine code. It is very important that the evaluation function diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index 6bd4484db68..eaa46acae94 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "compile_cache.h" #include <vespa/eval/eval/key_gen.h> #include <thread> diff --git a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp index 9a5f81c9282..30d21a987e0 100644 --- a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp +++ b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp @@ -1,12 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "compiled_function.h" -#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/eval/eval/param_usage.h> +#include <vespa/eval/eval/gbdt.h> #include <vespa/eval/eval/node_traverser.h> #include <vespa/eval/eval/check_type.h> #include <vespa/eval/eval/tensor_nodes.h> #include <vespa/vespalib/util/classname.h> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/util/approx.h> namespace vespalib { namespace eval { @@ -137,5 +139,20 @@ CompiledFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } +bool +CompiledFunction::should_use_lazy_params(const Function &function) +{ + if (gbdt::contains_gbdt(function.root(), 16)) { + return false; // contains gbdt + } + auto usage = vespalib::eval::check_param_usage(function); + for (double p_use: usage) { + if (!approx_equal(p_use, 1.0)) { + return true; // param not always used + } + } + return false; // all params always used +} + } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/eval/llvm/compiled_function.h b/eval/src/vespa/eval/eval/llvm/compiled_function.h index 912453f290a..13ef322e03b 100644 --- a/eval/src/vespa/eval/eval/llvm/compiled_function.h +++ b/eval/src/vespa/eval/eval/llvm/compiled_function.h @@ -63,6 +63,7 @@ public: } double estimate_cost_us(const std::vector<double> ¶ms, double budget = 5.0) const; static Function::Issues detect_issues(const Function &function); + static bool should_use_lazy_params(const Function &function); }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/llvm/deinline_forest.cpp b/eval/src/vespa/eval/eval/llvm/deinline_forest.cpp index bcfb94634bc..3b21af2146b 100644 --- a/eval/src/vespa/eval/eval/llvm/deinline_forest.cpp +++ b/eval/src/vespa/eval/eval/llvm/deinline_forest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "deinline_forest.h" namespace vespalib { diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index ac4c4e597ca..4a0aef82f6b 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -627,6 +627,15 @@ struct InitializeNativeTarget { llvm::InitializeNativeTargetAsmPrinter(); llvm::InitializeNativeTargetAsmParser(); } + ~InitializeNativeTarget() { + llvm::llvm_shutdown(); +#ifdef HAS_LLVM_DESTROY_STATIC_MUTEX + llvm::llvm_destroy_static_mutex(); +#endif +#ifdef HAS_LLVM_DESTROY_OPENED_HANDLES + llvm::llvm_destroy_opened_handles(); +#endif + } } initialize_native_target; std::recursive_mutex LLVMWrapper::_global_llvm_lock; diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index 72b48beb1a2..9943e507ce3 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <cmath> #include "operation.h" #include "value.h" #include "operation_visitor.h" +#include <cmath> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/param_usage.cpp b/eval/src/vespa/eval/eval/param_usage.cpp index c56bd953aa3..f7ca4d673d9 100644 --- a/eval/src/vespa/eval/eval/param_usage.cpp +++ b/eval/src/vespa/eval/eval/param_usage.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "param_usage.h" #include "function.h" #include "node_traverser.h" diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 71cf9b9b24f..7e6ef25ba02 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "simple_tensor.h" #include "simple_tensor_engine.h" #include "operation.h" #include <vespa/vespalib/objects/nbostream.h> #include <algorithm> +#include <cassert> namespace vespalib { namespace eval { @@ -416,14 +416,14 @@ public: }; struct Format { - bool is_sparse; - bool is_dense; - uint8_t tag; + bool is_sparse; + bool is_dense; + uint32_t tag; explicit Format(const TypeMeta &meta) : is_sparse(meta.mapped.size() > 0), is_dense((meta.indexed.size() > 0) || !is_sparse), tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0)) {} - explicit Format(uint8_t tag_in) + explicit Format(uint32_t tag_in) : is_sparse((tag_in & 0x1) != 0), is_dense((tag_in & 0x2) != 0), tag(tag_in) {} @@ -529,7 +529,7 @@ SimpleTensor::SimpleTensor(double value) { } -SimpleTensor::SimpleTensor(const ValueType &type_in, Cells &&cells_in) +SimpleTensor::SimpleTensor(const ValueType &type_in, Cells cells_in) : Tensor(SimpleTensorEngine::ref()), _type(type_in), _cells(std::move(cells_in)) @@ -689,7 +689,7 @@ SimpleTensor::encode(const SimpleTensor &tensor, nbostream &output) { TypeMeta meta(tensor.type()); Format format(meta); - output << format.tag; + output.putInt1_4Bytes(format.tag); encode_type(output, format, tensor.type(), meta); maybe_encode_num_blocks(output, meta, tensor.cells().size() / meta.block_size); View view(tensor, meta.mapped); @@ -705,7 +705,7 @@ SimpleTensor::encode(const SimpleTensor &tensor, nbostream &output) std::unique_ptr<SimpleTensor> SimpleTensor::decode(nbostream &input) { - Format format(input.readValue<uint8_t>()); + Format format(input.getInt1_4Bytes()); ValueType type = decode_type(input, format); TypeMeta meta(type); Builder builder(type); diff --git a/eval/src/vespa/eval/eval/simple_tensor.h b/eval/src/vespa/eval/eval/simple_tensor.h index 51b34813456..1c279f312c1 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.h +++ b/eval/src/vespa/eval/eval/simple_tensor.h @@ -77,7 +77,7 @@ private: public: SimpleTensor(); explicit SimpleTensor(double value); - SimpleTensor(const ValueType &type_in, Cells &&cells_in); + SimpleTensor(const ValueType &type_in, Cells cells_in); const ValueType &type() const { return _type; } const Cells &cells() const { return _cells; } std::unique_ptr<SimpleTensor> map(const std::function<double(double)> &function) const; diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index 216df3825e4..58a4e3b95b4 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/stringfmt.h> #include "simple_tensor_engine.h" #include "simple_tensor.h" -#include "operation.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/tensor.cpp b/eval/src/vespa/eval/eval/tensor.cpp index f79f14e5013..36741eea6db 100644 --- a/eval/src/vespa/eval/eval/tensor.cpp +++ b/eval/src/vespa/eval/eval/tensor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor.h" #include "tensor_engine.h" diff --git a/eval/src/vespa/eval/eval/tensor_engine.cpp b/eval/src/vespa/eval/eval/tensor_engine.cpp index 6ca06e68618..f495c94c83e 100644 --- a/eval/src/vespa/eval/eval/tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/tensor_engine.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_engine.h" namespace vespalib { diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 5750d90059f..b23880385e4 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_function.h" #include "value.h" #include "operation.h" diff --git a/eval/src/vespa/eval/eval/tensor_nodes.cpp b/eval/src/vespa/eval/eval/tensor_nodes.cpp index 5fd3b9dd627..400eda461b9 100644 --- a/eval/src/vespa/eval/eval/tensor_nodes.cpp +++ b/eval/src/vespa/eval/eval/tensor_nodes.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_nodes.h" #include "node_visitor.h" diff --git a/eval/src/vespa/eval/eval/tensor_spec.cpp b/eval/src/vespa/eval/eval/tensor_spec.cpp index ac59f0d4424..9f07f5fe7b2 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.cpp +++ b/eval/src/vespa/eval/eval/tensor_spec.cpp @@ -2,11 +2,33 @@ #include "tensor_spec.h" #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/slime/slime.h> #include <ostream> namespace vespalib { namespace eval { +namespace { + +TensorSpec::Address extract_address(const slime::Inspector &address) { + struct Extractor : slime::ObjectTraverser { + TensorSpec::Address address; + void field(const Memory &dimension, const slime::Inspector &label) override { + if (label.type().getId() == slime::STRING::ID) { + address.emplace(dimension.make_string(), TensorSpec::Label(label.asString().make_string())); + } else if (label.type().getId() == slime::LONG::ID) { + address.emplace(dimension.make_string(), TensorSpec::Label(label.asLong())); + } + } + }; + Extractor extractor; + address.traverse(extractor); + return extractor.address; +} + +} // namespace vespalib::eval::<unnamed> + + TensorSpec::TensorSpec(const vespalib::string &type_spec) : _type(type_spec), _cells() @@ -40,6 +62,38 @@ TensorSpec::to_string() const return out; } +void +TensorSpec::to_slime(slime::Cursor &tensor) const +{ + tensor.setString("type", _type); + slime::Cursor &cells = tensor.setArray("cells"); + for (const auto &my_cell: _cells) { + slime::Cursor &cell = cells.addObject(); + slime::Cursor &address = cell.setObject("address"); + for (const auto &label: my_cell.first) { + if (label.second.is_mapped()) { + address.setString(label.first, label.second.name); + } else { + address.setLong(label.first, label.second.index); + } + } + cell.setDouble("value", my_cell.second.value); + } +} + +TensorSpec +TensorSpec::from_slime(const slime::Inspector &tensor) +{ + TensorSpec spec(tensor["type"].asString().make_string()); + const slime::Inspector &cells = tensor["cells"]; + for (size_t i = 0; i < cells.entries(); ++i) { + const slime::Inspector &cell = cells[i]; + Address address = extract_address(cell["address"]); + spec.add(address, cell["value"].asDouble()); + } + return spec; +} + bool operator==(const TensorSpec &lhs, const TensorSpec &rhs) { diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 268b870aab9..d076821fa04 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -8,6 +8,14 @@ #include <map> namespace vespalib { + +namespace slime { + +class Cursor; +class Inspector; + +} // namespace vespalib::slime + namespace eval { /** @@ -59,12 +67,17 @@ public: TensorSpec & operator = (const TensorSpec &); ~TensorSpec(); TensorSpec &add(const Address &address, double value) { - _cells.emplace(address, value); + auto res = _cells.emplace(address, value); + if (!res.second) { + res.first->second.value += value; + } return *this; } const vespalib::string &type() const { return _type; } const Cells &cells() const { return _cells; } vespalib::string to_string() const; + void to_slime(slime::Cursor &tensor) const; + static TensorSpec from_slime(const slime::Inspector &tensor); }; bool operator==(const TensorSpec &lhs, const TensorSpec &rhs); diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index 594dfdd2210..ed83703dc4c 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "eval_spec.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/string_hash.h> diff --git a/eval/src/vespa/eval/eval/test/eval_spec.h b/eval/src/vespa/eval/eval/test/eval_spec.h index 4b9a244e598..e985b4bf5d4 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.h +++ b/eval/src/vespa/eval/eval/test/eval_spec.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <initializer_list> #include <cassert> +#include <vector> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 74dfeaebe31..931fb4e74fe 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -1,22 +1,27 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/testkit/test_kit.h> #include "tensor_conformance.h" -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/aggr.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/io/mapped_file_input.h> namespace vespalib { namespace eval { namespace test { namespace { +using slime::Cursor; +using slime::Inspector; +using slime::JsonFormat; + // Random access sequence of numbers struct Sequence { virtual double operator[](size_t i) const = 0; @@ -598,17 +603,40 @@ void verify_result(const Eval::Result &result, const TensorSpec &expect) { } } +uint8_t unhex(char c) { + if (c >= '0' && c <= '9') { + return (c - '0'); + } + if (c >= 'A' && c <= 'F') { + return ((c - 'A') + 10); + } + TEST_ERROR("bad hex char"); + return 0; +} + +nbostream extract_data(const Memory &hex_dump) { + nbostream data; + if ((hex_dump.size > 2) && (hex_dump.data[0] == '0') && (hex_dump.data[1] == 'x')) { + for (size_t i = 2; i < (hex_dump.size - 1); i += 2) { + data << uint8_t((unhex(hex_dump.data[i]) << 4) | unhex(hex_dump.data[i + 1])); + } + } + return data; +} + +bool is_same(const nbostream &a, const nbostream &b) { + return (Memory(a.peek(), a.size()) == Memory(b.peek(), b.size())); +} + // Test wrapper to avoid passing global test parameters around struct TestContext { + vespalib::string module_path; const TensorEngine &ref_engine; const TensorEngine &engine; - bool test_mixed_cases; - size_t skip_count; - TestContext(const TensorEngine &engine_in, bool test_mixed_cases_in) - : ref_engine(SimpleTensorEngine::ref()), engine(engine_in), - test_mixed_cases(test_mixed_cases_in), skip_count(0) {} + TestContext(const vespalib::string &module_path_in, const TensorEngine &engine_in) + : module_path(module_path_in), ref_engine(SimpleTensorEngine::ref()), engine(engine_in) {} std::unique_ptr<Tensor> tensor(const TensorSpec &spec) { auto result = engine.create(spec); @@ -616,13 +644,6 @@ struct TestContext { return result; } - bool mixed(size_t n) { - if (!test_mixed_cases) { - skip_count += n; - } - return test_mixed_cases; - } - //------------------------------------------------------------------------- void verify_create_type(const vespalib::string &type_spec) { @@ -637,10 +658,8 @@ struct TestContext { TEST_DO(verify_create_type("tensor(x{},y{})")); TEST_DO(verify_create_type("tensor(x[5])")); TEST_DO(verify_create_type("tensor(x[5],y[10])")); - if (mixed(2)) { - TEST_DO(verify_create_type("tensor(x{},y[10])")); - TEST_DO(verify_create_type("tensor(x[5],y{})")); - } + TEST_DO(verify_create_type("tensor(x{},y[10])")); + TEST_DO(verify_create_type("tensor(x[5],y{})")); } //------------------------------------------------------------------------- @@ -663,10 +682,8 @@ struct TestContext { TEST_DO(verify_equal(spec({x({"a"}),y({"a"})}, Seq({1})), spec({y({"a"}),x({"a"})}, Seq({1})))); TEST_DO(verify_equal(spec(x(3)), spec(x(3)))); TEST_DO(verify_equal(spec({x(1),y(1)}, Seq({1})), spec({y(1),x(1)}, Seq({1})))); - if (mixed(2)) { - TEST_DO(verify_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({y(1),x({"a"})}, Seq({1})))); - TEST_DO(verify_equal(spec({y({"a"}),x(1)}, Seq({1})), spec({x(1),y({"a"})}, Seq({1})))); - } + TEST_DO(verify_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({y(1),x({"a"})}, Seq({1})))); + TEST_DO(verify_equal(spec({y({"a"}),x(1)}, Seq({1})), spec({x(1),y({"a"})}, Seq({1})))); } //------------------------------------------------------------------------- @@ -696,12 +713,10 @@ struct TestContext { TEST_DO(verify_not_equal(spec(x(2), Seq({1,1}), Bits({1,0})), spec(x(2), Seq({1,1}), Bits({0,1})))); TEST_DO(verify_not_equal(spec(x(1), Seq({1})), spec({x(1),y(1)}, Seq({1})))); - if (mixed(3)) { - TEST_DO(verify_not_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({x({"a"}),y(1)}, Seq({2})))); - TEST_DO(verify_not_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({x({"b"}),y(1)}, Seq({1})))); - TEST_DO(verify_not_equal(spec({x(2),y({"a"})}, Seq({1}), Bits({1,0})), - spec({x(2),y({"a"})}, Seq({X,1}), Bits({0,1})))); - } + TEST_DO(verify_not_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({x({"a"}),y(1)}, Seq({2})))); + TEST_DO(verify_not_equal(spec({x({"a"}),y(1)}, Seq({1})), spec({x({"b"}),y(1)}, Seq({1})))); + TEST_DO(verify_not_equal(spec({x(2),y({"a"})}, Seq({1}), Bits({1,0})), + spec({x(2),y({"a"})}, Seq({X,1}), Bits({0,1})))); } //------------------------------------------------------------------------- @@ -717,12 +732,10 @@ struct TestContext { {x(3),y(5),z(7)}, {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})} + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})} }; - if (mixed(2 * 4)) { - layouts.push_back({x(3),y({"foo", "bar"}),z(7)}); - layouts.push_back({x({"a","b","c"}),y(5),z({"i","j","k","l"})}); - } for (const Layout &layout: layouts) { TensorSpec input = spec(layout, seq); for (const Domain &domain: layout) { @@ -757,12 +770,10 @@ struct TestContext { {x(3),y(5),z(7)}, {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})} + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})} }; - if (mixed(2 * 4)) { - layouts.push_back({x(3),y({"foo", "bar"}),z(7)}); - layouts.push_back({x({"a","b","c"}),y(5),z({"i","j","k","l"})}); - } for (const Layout &layout: layouts) { TensorSpec input = spec(layout, seq); for (const Domain &domain: layout) { @@ -804,12 +815,10 @@ struct TestContext { {x(3),y(5),z(7)}, {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})} + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})} }; - if (mixed(2)) { - layouts.push_back({x(3),y({"foo", "bar"}),z(7)}); - layouts.push_back({x({"a","b","c"}),y(5),z({"i","j","k","l"})}); - } for (const Layout &layout: layouts) { TEST_DO(verify_result(eval.eval(engine, spec(layout, seq)), spec(layout, OpSeq(seq, ref_op)))); } @@ -1074,14 +1083,10 @@ struct TestContext { {x({"a","b","c"})}, {y({"foo","bar","baz"})}, {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})} + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})} }; - if (mixed(2)) { - layouts.push_back({x(3),y({"foo", "bar"})}); - layouts.push_back({y({"foo", "bar"}),z(7)}); - layouts.push_back({x({"a","b","c"}),y(5)}); - layouts.push_back({y(5),z({"i","j","k","l"})}); - } ASSERT_TRUE((layouts.size() % 2) == 0); for (size_t i = 0; i < layouts.size(); i += 2) { TensorSpec lhs_input = spec(layouts[i], seq); @@ -1237,7 +1242,46 @@ struct TestContext { } } + void test_binary_format_spec(Cursor &test) { + Stash stash; + TensorSpec spec = TensorSpec::from_slime(test["tensor"]); + const Inspector &binary = test["binary"]; + EXPECT_GREATER(binary.entries(), 0u); + nbostream encoded; + engine.encode(make_value(engine, spec, stash), encoded, stash); + test.setData("encoded", Memory(encoded.peek(), encoded.size())); + bool matched_encode = false; + for (size_t i = 0; i < binary.entries(); ++i) { + nbostream data = extract_data(binary[i].asString()); + matched_encode = (matched_encode || is_same(encoded, data)); + TEST_DO(verify_result(Eval::Result(engine.decode(data, stash)), spec)); + EXPECT_EQUAL(data.size(), 0u); + } + EXPECT_TRUE(matched_encode); + } + + void test_binary_format_spec() { + vespalib::string path = module_path; + path.append("src/apps/make_tensor_binary_format_test_spec/test_spec.json"); + MappedFileInput file(path); + Slime slime; + EXPECT_TRUE(file.valid()); + EXPECT_EQUAL(JsonFormat::decode(file, slime), file.get().size); + int64_t num_tests = slime.get()["num_tests"].asLong(); + Cursor &tests = slime.get()["tests"]; + EXPECT_GREATER(num_tests, 0u); + EXPECT_EQUAL(size_t(num_tests), tests.entries()); + for (size_t i = 0; i < tests.entries(); ++i) { + size_t fail_cnt = TEST_MASTER.getProgress().failCnt; + TEST_DO(test_binary_format_spec(tests[i])); + if (TEST_MASTER.getProgress().failCnt > fail_cnt) { + fprintf(stderr, "failed:\n%s", tests[i].toString().c_str()); + } + } + } + void test_binary_format() { + TEST_DO(test_binary_format_spec()); TEST_DO(verify_encode_decode(spec(42))); TEST_DO(verify_encode_decode(spec({x(3)}, N()))); TEST_DO(verify_encode_decode(spec({x(3),y(5)}, N()))); @@ -1245,10 +1289,8 @@ struct TestContext { TEST_DO(verify_encode_decode(spec({x({"a","b","c"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, N()))); - if (mixed(2)) { - TEST_DO(verify_encode_decode(spec({x(3),y({"foo", "bar"}),z(7)}, N()))); - TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y(5),z({"i","j","k","l"})}, N()))); - } + TEST_DO(verify_encode_decode(spec({x(3),y({"foo", "bar"}),z(7)}, N()))); + TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y(5),z({"i","j","k","l"})}, N()))); } //------------------------------------------------------------------------- @@ -1271,13 +1313,11 @@ struct TestContext { } // namespace vespalib::eval::test::<unnamed> void -TensorConformance::run_tests(const TensorEngine &engine, bool test_mixed_cases) +TensorConformance::run_tests(const vespalib::string &module_path, const TensorEngine &engine) { - TestContext ctx(engine, test_mixed_cases); + TestContext ctx(module_path, engine); + fprintf(stderr, "module path: '%s'\n", ctx.module_path.c_str()); ctx.run_tests(); - if (ctx.skip_count > 0) { - fprintf(stderr, "WARNING: skipped %zu mixed test cases\n", ctx.skip_count); - } } } // namespace vespalib::eval::test diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.h b/eval/src/vespa/eval/eval/test/tensor_conformance.h index 054a1bb8faa..b9d4940bdba 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.h +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/eval/eval/tensor_engine.h> +#include <vespa/vespalib/stllike/string.h> namespace vespalib { namespace eval { @@ -13,7 +14,7 @@ namespace test { * implementations of the TensorEngine interface. **/ struct TensorConformance { - static void run_tests(const TensorEngine &engine, bool test_mixed_cases); + static void run_tests(const vespalib::string &module_path, const TensorEngine &engine); }; } // namespace vespalib::eval::test diff --git a/eval/src/vespa/eval/eval/value.cpp b/eval/src/vespa/eval/eval/value.cpp index 3db6f500e10..6b365d648ef 100644 --- a/eval/src/vespa/eval/eval/value.cpp +++ b/eval/src/vespa/eval/eval/value.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "value.h" #include "operation_visitor.h" #include "tensor_engine.h" diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index 65c36ca0ef2..05fdb3699bd 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> - #include "constant_tensor_loader.h" -#include <set> -#include <vespa/vespalib/data/slime/slime.h> #include <vespa/eval/eval/tensor.h> #include <vespa/eval/eval/tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/data/lz4_input_decoder.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <set> +#include <vespa/log/log.h> LOG_SETUP(".vespalib.eval.value_cache.constant_tensor_loader"); namespace vespalib { diff --git a/eval/src/vespa/eval/eval/value_cache/constant_value_cache.cpp b/eval/src/vespa/eval/eval/value_cache/constant_value_cache.cpp index fdda42fd0a5..0eebff6b84b 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_value_cache.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_value_cache.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "constant_value_cache.h" +#include <cassert> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index a038ee46583..84972879320 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "value_type.h" #include "value_type_spec.h" diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index 6d3aabef142..9cafd7e53c2 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "value_type.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <algorithm> #include "value_type_spec.h" +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> +#include <algorithm> namespace vespalib { namespace eval { diff --git a/eval/src/vespa/eval/eval/vm_forest.cpp b/eval/src/vespa/eval/eval/vm_forest.cpp index 3cf5546f7eb..284cdc514f3 100644 --- a/eval/src/vespa/eval/eval/vm_forest.cpp +++ b/eval/src/vespa/eval/eval/vm_forest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "gbdt.h" #include "vm_forest.h" #include <vespa/eval/eval/basic_nodes.h> diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 3b4111cc723..649b90de92e 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_add_library(eval_tensor OBJECT tensor_apply.cpp tensor_factory.cpp tensor_mapper.cpp + wrapped_simple_tensor.cpp ) diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 571fe8a293e..a2aca0b0e67 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -1,17 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "default_tensor_engine.h" -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/operation_visitor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include "tensor.h" +#include "default_tensor.h" +#include "wrapped_simple_tensor.h" #include "serialization/typed_binary_format.h" #include "dense/dense_tensor.h" #include "dense/dense_tensor_builder.h" #include "dense/dense_tensor_function_compiler.h" -#include "default_tensor.h" +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/operation_visitor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <cassert> + namespace vespalib { namespace tensor { @@ -25,19 +27,47 @@ using TensorSpec = eval::TensorSpec; namespace { -const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { - if (tensor->getType().is_tensor()) { - return stash.create<TensorValue>(std::move(tensor)); +const eval::TensorEngine &simple_engine() { return eval::SimpleTensorEngine::ref(); } +const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); } + +// map tensors to simple tensors before fall-back evaluation + +const eval::SimpleTensor &to_simple(const eval::Tensor &tensor, Stash &stash) { + if (auto wrapped = dynamic_cast<const WrappedSimpleTensor *>(&tensor)) { + return wrapped->get(); } - return stash.create<DoubleValue>(tensor->sum()); + TensorSpec spec = tensor.engine().to_spec(tensor); + using PTR = std::unique_ptr<eval::SimpleTensor>; + return *stash.create<PTR>(eval::SimpleTensor::create(spec)); } -const Tensor &to_tensor(const Value &value, Stash &stash) { +const Value &to_simple(const Value &value, Stash &stash) { + if (auto tensor = value.as_tensor()) { + return stash.create<TensorValue>(to_simple(*tensor, stash)); + } + return value; +} + +// map tensors to default tensors after fall-back evaluation + +const Value &to_default(const Value &value, Stash &stash) { if (auto tensor = value.as_tensor()) { - return static_cast<const Tensor &>(*tensor); + if (auto simple = dynamic_cast<const eval::SimpleTensor *>(tensor)) { + if (!Tensor::supported({simple->type()})) { + return stash.create<TensorValue>(std::make_unique<WrappedSimpleTensor>(*simple)); + } + } + TensorSpec spec = tensor->engine().to_spec(*tensor); + return stash.create<TensorValue>(default_engine().create(spec)); + } + return value; +} + +const Value &to_value(std::unique_ptr<Tensor> tensor, Stash &stash) { + if (tensor->getType().is_tensor()) { + return stash.create<TensorValue>(std::move(tensor)); } - return stash.create<DenseTensor>(ValueType::double_type(), - std::vector<double>({value.as_double()})); + return stash.create<DoubleValue>(tensor->sum()); } } // namespace vespalib::tensor::<unnamed> @@ -59,9 +89,6 @@ DefaultTensorEngine::equal(const Tensor &a, const Tensor &b) const assert(&b.engine() == this); const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(a); const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(b); - if (my_a.getType().type() != my_b.getType().type()) { - return false; - } return my_a.equals(my_b); } @@ -108,7 +135,7 @@ DefaultTensorEngine::create(const TensorSpec &spec) const } } if (is_dense && is_sparse) { - return DefaultTensor::builder().build(); + return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::create(spec)); } else if (is_dense) { DenseTensorBuilder builder; std::map<vespalib::string,DenseTensorBuilder::Dimension> dimension_map; @@ -145,6 +172,9 @@ DefaultTensorEngine::reduce(const Tensor &tensor, const BinaryOperation &op, con { assert(&tensor.engine() == this); const tensor::Tensor &my_tensor = static_cast<const tensor::Tensor &>(tensor); + if (!tensor::Tensor::supported({my_tensor.getType()})) { + return to_default(simple_engine().reduce(to_simple(my_tensor, stash), op, dimensions, stash), stash); + } IsAddOperation check; op.accept(check); tensor::Tensor::UP result; @@ -180,6 +210,9 @@ DefaultTensorEngine::map(const UnaryOperation &op, const Tensor &a, Stash &stash { assert(&a.engine() == this); const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(a); + if (!tensor::Tensor::supported({my_a.getType()})) { + return to_default(simple_engine().map(op, to_simple(my_a, stash), stash), stash); + } CellFunctionAdapter cell_function(op); return to_value(my_a.apply(cell_function), stash); } @@ -227,8 +260,8 @@ DefaultTensorEngine::apply(const BinaryOperation &op, const Tensor &a, const Ten assert(&b.engine() == this); const tensor::Tensor &my_a = static_cast<const tensor::Tensor &>(a); const tensor::Tensor &my_b = static_cast<const tensor::Tensor &>(b); - if (my_a.getType().type() != my_b.getType().type()) { - return stash.create<ErrorValue>(); + if (!tensor::Tensor::supported({my_a.getType(), my_b.getType()})) { + return to_default(simple_engine().apply(op, to_simple(my_a, stash), to_simple(my_b, stash), stash), stash); } TensorOperationOverride tensor_override(my_a, my_b); op.accept(tensor_override); @@ -241,37 +274,14 @@ DefaultTensorEngine::apply(const BinaryOperation &op, const Tensor &a, const Ten //----------------------------------------------------------------------------- -namespace { - -const eval::TensorEngine &simple_engine() { return eval::SimpleTensorEngine::ref(); } -const eval::TensorEngine &default_engine() { return DefaultTensorEngine::ref(); } - -// map tensors to simple tensors before fall-back evaluation -const Value &to_simple(const Value &value, Stash &stash) { - if (auto tensor = value.as_tensor()) { - TensorSpec spec = tensor->engine().to_spec(*tensor); - return stash.create<TensorValue>(simple_engine().create(spec)); - } - return value; -} - -// map tensors to default tensors after fall-back evaluation -const Value &to_default(const Value &value, Stash &stash) { - if (auto tensor = value.as_tensor()) { - TensorSpec spec = tensor->engine().to_spec(*tensor); - return stash.create<TensorValue>(default_engine().create(spec)); - } - return value; -} - -} // namespace vespalib::tensor::<unnamed> - -//----------------------------------------------------------------------------- - void -DefaultTensorEngine::encode(const Value &value, nbostream &output, Stash &stash) const +DefaultTensorEngine::encode(const Value &value, nbostream &output, Stash &) const { - TypedBinaryFormat::serialize(output, to_tensor(value, stash)); + if (auto tensor = value.as_tensor()) { + TypedBinaryFormat::serialize(output, static_cast<const tensor::Tensor &>(*tensor)); + } else { + TypedBinaryFormat::serialize(output, DenseTensor(ValueType::double_type(), {value.as_double()})); + } } const Value & diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index ef400193427..835e86fbb28 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_dot_product_function.h" #include "dense_tensor.h" #include "dense_tensor_view.h" diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp index 1421aed887b..30bc395c1aa 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.h b/eval/src/vespa/eval/tensor/dense/dense_tensor.h index f4d6877bf61..91450ce3ce4 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.h @@ -28,6 +28,7 @@ private: public: DenseTensor(); + ~DenseTensor() {} DenseTensor(const eval::ValueType &type_in, const Cells &cells_in); DenseTensor(const eval::ValueType &type_in, diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp index a85a65ed8a9..1112362e752 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_address_combiner.h" #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp index a00537ffa50..3c6e182a143 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp @@ -2,6 +2,7 @@ #include "dense_tensor_builder.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <cassert> using vespalib::IllegalArgumentException; diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp index 84311e47e5a..cf36030f0ea 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_cells_iterator.h" namespace vespalib { diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp index cac20a51c01..f602c1be46e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_dot_product_function.h" #include "dense_tensor_function_compiler.h" #include <vespa/eval/eval/operation_visitor.h> diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp index ed532ab14d6..f3e7a6109c6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.hpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dense_tensor_reduce.h" +#include <cassert> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 31e260f6f4e..97bb634a862 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_view.h" #include "dense_tensor_apply.hpp" #include "dense_tensor_reduce.hpp" diff --git a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp index e98bc0261d8..23abcf57da4 100644 --- a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "direct_dense_tensor_builder.h" +#include <cassert> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 484db417dc9..165c6823ac2 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_binary_format.h" #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/vespalib/objects/nbostream.h> - +#include <cassert> using vespalib::nbostream; diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h index cb080b6b4ee..df7eb12ccfd 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h @@ -2,6 +2,7 @@ #pragma once +#include <memory> namespace vespalib { class nbostream; diff --git a/eval/src/vespa/eval/tensor/serialization/format.txt b/eval/src/vespa/eval/tensor/serialization/format.txt index 8c5d3b331d2..780f88af01a 100644 --- a/eval/src/vespa/eval/tensor/serialization/format.txt +++ b/eval/src/vespa/eval/tensor/serialization/format.txt @@ -8,7 +8,7 @@ effortlessly as possible with both existing formats. //----------------------------------------------------------------------------- -byte: type (1:sparse, 2:dense, 3:mixed) +1_4_int: type (1:sparse, 2:dense, 3:mixed) bit 0 -> 'sparse' bit 1 -> 'dense' (mixed tensors are tagged as both 'sparse' and 'dense') diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp index 7bb1477e7a3..fe2270cdb17 100644 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "slime_binary_format.h" #include <vespa/eval/tensor/types.h> #include <vespa/eval/tensor/tensor.h> diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h index 0af80c5de61..e6d7e8a7258 100644 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace vespalib { class Slime; diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp index 50d4a91965e..2658df78f31 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_binary_format.h" #include <vespa/eval/tensor/types.h> #include <vespa/eval/tensor/tensor.h> @@ -8,14 +7,13 @@ #include <vespa/eval/tensor/tensor_visitor.h> #include <vespa/vespalib/objects/nbostream.h> #include <sstream> - +#include <cassert> using vespalib::nbostream; namespace vespalib { namespace tensor { - namespace { vespalib::string undefinedLabel(""); diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index e3800b3d7ad..c4c549f7f05 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "typed_binary_format.h" #include "sparse_binary_format.h" #include "dense_binary_format.h" @@ -8,6 +7,8 @@ #include <vespa/eval/tensor/default_tensor.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/tensor/wrapped_simple_tensor.h> using vespalib::nbostream; @@ -18,10 +19,11 @@ namespace tensor { void TypedBinaryFormat::serialize(nbostream &stream, const Tensor &tensor) { - const DenseTensor *denseTensor = dynamic_cast<const DenseTensor *>(&tensor); - if (denseTensor != nullptr) { + if (auto denseTensor = dynamic_cast<const DenseTensor *>(&tensor)) { stream.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); DenseBinaryFormat::serialize(stream, *denseTensor); + } else if (auto wrapped = dynamic_cast<const WrappedSimpleTensor *>(&tensor)) { + eval::SimpleTensor::encode(wrapped->get(), stream); } else { stream.putInt1_4Bytes(SPARSE_BINARY_FORMAT_TYPE); SparseBinaryFormat::serialize(stream, tensor); @@ -32,6 +34,7 @@ TypedBinaryFormat::serialize(nbostream &stream, const Tensor &tensor) std::unique_ptr<Tensor> TypedBinaryFormat::deserialize(nbostream &stream) { + auto read_pos = stream.rp(); auto formatId = stream.getInt1_4Bytes(); if (formatId == SPARSE_BINARY_FORMAT_TYPE) { DefaultTensor::builder builder; @@ -41,6 +44,10 @@ TypedBinaryFormat::deserialize(nbostream &stream) if (formatId == DENSE_BINARY_FORMAT_TYPE) { return DenseBinaryFormat::deserialize(stream); } + if (formatId == MIXED_BINARY_FORMAT_TYPE) { + stream.adjustReadPos(read_pos - stream.rp()); + return std::make_unique<WrappedSimpleTensor>(eval::SimpleTensor::decode(stream)); + } abort(); } diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h index 772f820ffc5..bfe34b18e6d 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.h @@ -2,6 +2,9 @@ #pragma once +#include <memory> +#include <cstdint> + namespace vespalib { class nbostream; @@ -18,6 +21,7 @@ class TypedBinaryFormat { static constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u; static constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u; + static constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u; public: static void serialize(nbostream &stream, const Tensor &tensor); static std::unique_ptr<Tensor> deserialize(nbostream &stream); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp index 82ec52e0108..d95b1c6ab42 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_tensor_address_combiner.h" #include "sparse_tensor_address_decoder.h" #include <vespa/eval/eval/value_type.h> +#include <cassert> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp index b7df157c158..c85f520ba5a 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sparse_tensor_unsorted_address_builder.h" #include "sparse_tensor_address_builder.h" #include <vespa/eval/eval/value_type.h> -#include <algorithm> +#include <cassert> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/tensor.cpp b/eval/src/vespa/eval/tensor/tensor.cpp index 11ef2b0ad00..5c9713fa1b5 100644 --- a/eval/src/vespa/eval/tensor/tensor.cpp +++ b/eval/src/vespa/eval/tensor/tensor.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor.h" -#include <sstream> #include "default_tensor_engine.h" +#include <sstream> namespace vespalib { namespace tensor { @@ -13,6 +12,21 @@ Tensor::Tensor() { } +bool +Tensor::supported(TypeList types) +{ + bool sparse = false; + bool dense = false; + for (const eval::ValueType &type: types) { + dense = (dense || type.is_double()); + for (const auto &dim: type.dimensions()) { + dense = (dense || dim.is_indexed()); + sparse = (sparse || dim.is_mapped()); + } + } + return (dense != sparse); +} + std::ostream & operator<<(std::ostream &out, const Tensor &value) { diff --git a/eval/src/vespa/eval/tensor/tensor.h b/eval/src/vespa/eval/tensor/tensor.h index 379dedcb2a2..856f01d15b9 100644 --- a/eval/src/vespa/eval/tensor/tensor.h +++ b/eval/src/vespa/eval/tensor/tensor.h @@ -50,6 +50,9 @@ struct Tensor : public eval::Tensor virtual Tensor::UP clone() const = 0; virtual eval::TensorSpec toSpec() const = 0; virtual void accept(TensorVisitor &visitor) const = 0; + + using TypeList = std::initializer_list<std::reference_wrapper<const eval::ValueType>>; + static bool supported(TypeList types); }; std::ostream &operator<<(std::ostream &out, const Tensor &value); diff --git a/eval/src/vespa/eval/tensor/tensor_factory.cpp b/eval/src/vespa/eval/tensor/tensor_factory.cpp index 84f7c0cc705..58810b33f49 100644 --- a/eval/src/vespa/eval/tensor/tensor_factory.cpp +++ b/eval/src/vespa/eval/tensor/tensor_factory.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor.h" #include "tensor_factory.h" #include "tensor_builder.h" diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.cpp b/eval/src/vespa/eval/tensor/tensor_mapper.cpp index 939e6fdfa26..15dfb1db83a 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.cpp +++ b/eval/src/vespa/eval/tensor/tensor_mapper.cpp @@ -5,16 +5,21 @@ #include "tensor_visitor.h" #include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> #include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/eval/simple_tensor.h> #include "tensor_address_element_iterator.h" #include "default_tensor.h" +#include "wrapped_simple_tensor.h" using vespalib::eval::ValueType; +using vespalib::eval::TensorSpec; namespace vespalib { namespace tensor { namespace { +//----------------------------------------------------------------------------- + template <class TensorT> class SparseTensorMapper : public TensorVisitor { @@ -96,6 +101,8 @@ SparseTensorMapper<TensorT>::map(const Tensor &tensor, return mapper.build(); } +//----------------------------------------------------------------------------- + static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t BAD_ADDRESS = std::numeric_limits<uint32_t>::max(); @@ -113,7 +120,9 @@ uint32_t mapLabelToNumber(vespalib::stringref label) { return result; } -class DenseTensorTypeMapper : public TensorVisitor +//----------------------------------------------------------------------------- + +class TensorTypeMapper : public TensorVisitor { ValueType _type; std::vector<ValueType::Dimension> _dimensions; @@ -123,8 +132,8 @@ class DenseTensorTypeMapper : public TensorVisitor virtual void visit(const TensorAddress &address, double value) override; - DenseTensorTypeMapper(const ValueType &type); - ~DenseTensorTypeMapper(); + TensorTypeMapper(const ValueType &type); + ~TensorTypeMapper(); ValueType build(); public: @@ -132,16 +141,18 @@ public: }; bool -DenseTensorTypeMapper::addressOK(const TensorAddress &address) +TensorTypeMapper::addressOK(const TensorAddress &address) { TensorAddressElementIterator<TensorAddress> addressIterator(address); auto dimIterator = _dimensions.begin(); for (const auto &dimension : _type.dimensions()) { if (addressIterator.skipToDimension(dimension.name)) { - uint32_t label = mapLabelToNumber(addressIterator.label()); - if (label == BAD_LABEL || - (dimension.is_bound() && label >= dimIterator->size)) { - return false; + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label == BAD_LABEL || + (dimension.is_bound() && label >= dimIterator->size)) { + return false; + } } addressIterator.next(); } @@ -153,17 +164,19 @@ DenseTensorTypeMapper::addressOK(const TensorAddress &address) void -DenseTensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) +TensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) { TensorAddressElementIterator<TensorAddress> addressIterator(address); auto dimIterator = _dimensions.begin(); for (const auto &dimension : _type.dimensions()) { if (addressIterator.skipToDimension(dimension.name)) { - uint32_t label = mapLabelToNumber(addressIterator.label()); - if (label != BAD_LABEL && - !dimension.is_bound() && - label >= dimIterator->size) { - dimIterator->size = label + 1; + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if (label != BAD_LABEL && + !dimension.is_bound() && + label >= dimIterator->size) { + dimIterator->size = label + 1; + } } addressIterator.next(); } @@ -173,7 +186,7 @@ DenseTensorTypeMapper::expandUnboundDimensions(const TensorAddress &address) } void -DenseTensorTypeMapper::visit(const TensorAddress &address, double value) +TensorTypeMapper::visit(const TensorAddress &address, double value) { (void) value; if (addressOK(address)) { @@ -181,37 +194,42 @@ DenseTensorTypeMapper::visit(const TensorAddress &address, double value) } } -DenseTensorTypeMapper::DenseTensorTypeMapper(const ValueType &type) +TensorTypeMapper::TensorTypeMapper(const ValueType &type) : _type(type), _dimensions(type.dimensions()) { for (auto &dimension : _dimensions) { - if (!dimension.is_bound()) - dimension.size = 1; + if (dimension.is_indexed()) { + if (!dimension.is_bound()) { + dimension.size = 1; + } + } } } -DenseTensorTypeMapper::~DenseTensorTypeMapper() +TensorTypeMapper::~TensorTypeMapper() { } ValueType -DenseTensorTypeMapper::build() +TensorTypeMapper::build() { return ValueType::tensor_type(std::move(_dimensions)); } ValueType -DenseTensorTypeMapper::map(const Tensor &tensor, const ValueType &type) +TensorTypeMapper::map(const Tensor &tensor, const ValueType &type) { - DenseTensorTypeMapper mapper(type); + TensorTypeMapper mapper(type); tensor.accept(mapper); return mapper.build(); } +//----------------------------------------------------------------------------- + class DenseTensorMapper : public TensorVisitor { - eval::ValueType _type; + ValueType _type; DenseTensor::Cells _cells; uint32_t mapAddressToIndex(const TensorAddress &address); @@ -283,12 +301,77 @@ std::unique_ptr<Tensor> DenseTensorMapper::map(const Tensor &tensor, const ValueType &type) { DenseTensorMapper mapper(type.is_abstract() ? - DenseTensorTypeMapper::map(tensor, type) : + TensorTypeMapper::map(tensor, type) : type); tensor.accept(mapper); return mapper.build(); } +//----------------------------------------------------------------------------- + +class WrappedTensorMapper : public TensorVisitor +{ + using Label = TensorSpec::Label; + + ValueType _type; + TensorSpec _spec; + + WrappedTensorMapper(const ValueType &type) + : _type(type), _spec(type.to_spec()) {} + ~WrappedTensorMapper() {} + + void visit(const TensorAddress &address, double value) override; + + std::unique_ptr<Tensor> build() { + auto tensor = eval::SimpleTensor::create(_spec); + return std::make_unique<WrappedSimpleTensor>(std::move(tensor)); + } + +public: + static std::unique_ptr<Tensor> + map(const Tensor &tensor, const ValueType &type); +}; + +void +WrappedTensorMapper::visit(const TensorAddress &address, double value) +{ + TensorSpec::Address addr; + TensorAddressElementIterator<TensorAddress> addressIterator(address); + for (const auto &dimension: _type.dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { + if (dimension.is_indexed()) { + uint32_t label = mapLabelToNumber(addressIterator.label()); + if ((label == BAD_LABEL) || (label >= dimension.size)) { + return; // bad address; ignore cell + } + addr.emplace(dimension.name, label); + } else { + addr.emplace(dimension.name, addressIterator.label()); + } + addressIterator.next(); + } else { + if (dimension.is_indexed()) { + addr.emplace(dimension.name, size_t(0)); + } else { + addr.emplace(dimension.name, vespalib::string()); + } + } + } + _spec.add(addr, value); +} + +std::unique_ptr<Tensor> +WrappedTensorMapper::map(const Tensor &tensor, const ValueType &type) +{ + WrappedTensorMapper mapper(type.is_abstract() ? + TensorTypeMapper::map(tensor, type) : + type); + tensor.accept(mapper); + return mapper.build(); +} + +//----------------------------------------------------------------------------- + } // namespace vespalib::tensor::<anonymous> TensorMapper::TensorMapper(const ValueType &type) @@ -316,6 +399,13 @@ TensorMapper::mapToDense(const Tensor &tensor, const ValueType &type) } std::unique_ptr<Tensor> +TensorMapper::mapToWrapped(const Tensor &tensor, const ValueType &type) +{ + assert(!type.dimensions().empty()); + return WrappedTensorMapper::map(tensor, type); +} + +std::unique_ptr<Tensor> TensorMapper::map(const Tensor &tensor) const { if (_type.is_sparse()) { @@ -323,7 +413,7 @@ TensorMapper::map(const Tensor &tensor) const } else if (_type.is_dense()) { return mapToDense(tensor, _type); } else { - return std::unique_ptr<Tensor>(); + return mapToWrapped(tensor, _type); } } diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.h b/eval/src/vespa/eval/tensor/tensor_mapper.h index 09d8a6208a3..02b0a30e185 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.h +++ b/eval/src/vespa/eval/tensor/tensor_mapper.h @@ -35,6 +35,9 @@ public: static std::unique_ptr<Tensor> mapToDense(const Tensor &tensor, const eval::ValueType &type); + static std::unique_ptr<Tensor> + mapToWrapped(const Tensor &tensor, const eval::ValueType &type); + std::unique_ptr<Tensor> map(const Tensor &tensor) const; }; diff --git a/eval/src/vespa/eval/tensor/types.h b/eval/src/vespa/eval/tensor/types.h index 7bdb37b8ac2..f1b108a0442 100644 --- a/eval/src/vespa/eval/tensor/types.h +++ b/eval/src/vespa/eval/tensor/types.h @@ -3,8 +3,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <map> #include <vespa/vespalib/stllike/hash_set.h> +#include <vector> +#include <map> namespace vespalib { namespace tensor { diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp new file mode 100644 index 00000000000..77cb517c063 --- /dev/null +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -0,0 +1,145 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "wrapped_simple_tensor.h" +#include "tensor_address_builder.h" +#include "tensor_visitor.h" +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/vespalib/util/stringfmt.h> + +namespace vespalib::tensor { + +bool +WrappedSimpleTensor::equals(const Tensor &arg) const +{ + if (auto other = dynamic_cast<const WrappedSimpleTensor *>(&arg)) { + return eval::SimpleTensor::equal(_tensor, other->_tensor); + } + return false; +} + +vespalib::string +WrappedSimpleTensor::toString() const +{ + return eval::SimpleTensorEngine::ref().to_string(_tensor); +} + +eval::TensorSpec +WrappedSimpleTensor::toSpec() const +{ + return eval::SimpleTensorEngine::ref().to_spec(_tensor); +} + +double +WrappedSimpleTensor::sum() const +{ + double result = 0.0; + for (const auto &cell: _tensor.cells()) { + result += cell.value; + } + return result; +} + +void +WrappedSimpleTensor::accept(TensorVisitor &visitor) const +{ + TensorAddressBuilder addr; + const auto &dimensions = _tensor.type().dimensions(); + for (const auto &cell: _tensor.cells()) { + addr.clear(); + for (size_t i = 0; i < dimensions.size(); ++i) { + if (dimensions[i].is_indexed()) { + addr.add(dimensions[i].name, make_string("%zu", cell.address[i].index)); + } else { + addr.add(dimensions[i].name, cell.address[i].name); + } + } + visitor.visit(addr.build(), cell.value); + } +} + +void +WrappedSimpleTensor::print(std::ostream &out) const +{ + out << toString(); +} + +Tensor::UP +WrappedSimpleTensor::clone() const +{ + auto tensor = std::make_unique<eval::SimpleTensor>(_tensor.type(), _tensor.cells()); + return std::make_unique<WrappedSimpleTensor>(std::move(tensor)); +} + +//----------------------------------------------------------------------------- + +Tensor::UP +WrappedSimpleTensor::add(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::subtract(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::multiply(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::min(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::max(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::match(const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::apply(const CellFunction &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::sum(const vespalib::string &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::apply(const eval::BinaryOperation &, const Tensor &) const +{ + abort(); + return Tensor::UP(); +} + +Tensor::UP +WrappedSimpleTensor::reduce(const eval::BinaryOperation &, const std::vector<vespalib::string> &) const +{ + abort(); + return Tensor::UP(); +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h new file mode 100644 index 00000000000..1f6c22254df --- /dev/null +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "tensor.h" +#include <vespa/eval/eval/simple_tensor.h> + +namespace vespalib::tensor { + +/** + * A thin wrapper around a SimpleTensor (tensor reference + * implementation) to be used as fallback for tensors with data + * layouts not supported by the default tensor implementation. + * + * Tensor implementation class is currently inferred from its value + * type. Consider adding explicit tagging to the tensor::Tensor + * default implementation top-level class in the future. + **/ +class WrappedSimpleTensor : public Tensor +{ +private: + std::unique_ptr<eval::SimpleTensor> _space; + const eval::SimpleTensor &_tensor; +public: + explicit WrappedSimpleTensor(const eval::SimpleTensor &tensor) + : _space(), _tensor(tensor) {} + explicit WrappedSimpleTensor(std::unique_ptr<eval::SimpleTensor> tensor) + : _space(std::move(tensor)), _tensor(*_space) {} + ~WrappedSimpleTensor() {} + const eval::SimpleTensor &get() const { return _tensor; } + const eval::ValueType &getType() const override { return _tensor.type(); } + bool equals(const Tensor &arg) const override; + vespalib::string toString() const override; + eval::TensorSpec toSpec() const override; + double sum() const override; + void accept(TensorVisitor &visitor) const override; + void print(std::ostream &out) const override; + Tensor::UP clone() const override; + // functions below should not be used for this implementation + Tensor::UP add(const Tensor &) const override; + Tensor::UP subtract(const Tensor &) const override; + Tensor::UP multiply(const Tensor &) const override; + Tensor::UP min(const Tensor &) const override; + Tensor::UP max(const Tensor &) const override; + Tensor::UP match(const Tensor &) const override; + Tensor::UP apply(const CellFunction &) const override; + Tensor::UP sum(const vespalib::string &) const override; + Tensor::UP apply(const eval::BinaryOperation &, const Tensor &) const override; + Tensor::UP reduce(const eval::BinaryOperation &, const std::vector<vespalib::string> &) const override; +}; + +} // namespace vespalib::tensor diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp index ae498595e35..04376826139 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000 Fast Search & Transfer -#include <vespa/fastos/fastos.h> -#include <vespa/fastlib/io/bufferedfile.h> +#include "bufferedfile.h" namespace { diff --git a/fastlib/src/vespa/fastlib/io/bufferedinputstream.cpp b/fastlib/src/vespa/fastlib/io/bufferedinputstream.cpp index 26bab632aeb..dd212ddfd08 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedinputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedinputstream.cpp @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "bufferedinputstream.h" +#include <cstring> +#include <cstdint> Fast_BufferedInputStream::Fast_BufferedInputStream(Fast_InputStream &in, size_t bufferSize) : Fast_FilterInputStream(in), _buffer(new char[bufferSize]), - _bufferSize((_buffer != NULL) ? bufferSize : 0), + _bufferSize((_buffer != nullptr) ? bufferSize : 0), _bufferUsed(0), _bufferRead(0), _nextWillFail(false) diff --git a/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp b/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp index b025a5dba25..7377c666cf4 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp @@ -11,11 +11,9 @@ * ALL RIGHTS RESERVED * ******************************************************************************/ -#include <vespa/fastos/fastos.h> -#include "bufferedoutputstream.h" - - +#include "bufferedoutputstream.h" +#include <cstring> Fast_BufferedOutputStream::Fast_BufferedOutputStream(Fast_OutputStream &out, size_t bufferSize) @@ -33,7 +31,7 @@ Fast_BufferedOutputStream::Fast_BufferedOutputStream(Fast_OutputStream &out, Fast_BufferedOutputStream::~Fast_BufferedOutputStream(void) { delete [] _buffer; -}; +} diff --git a/fastlib/src/vespa/fastlib/io/fileinputstream.cpp b/fastlib/src/vespa/fastlib/io/fileinputstream.cpp index d501a752cfe..2d00935df79 100644 --- a/fastlib/src/vespa/fastlib/io/fileinputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/fileinputstream.cpp @@ -11,10 +11,8 @@ * ALL RIGHTS RESERVED * ******************************************************************************/ -#include <vespa/fastos/fastos.h> -#include <limits.h> -#include <vespa/fastlib/io/fileinputstream.h> +#include "fileinputstream.h" Fast_FileInputStream::Fast_FileInputStream(const char *fileName) : _theFile(new FastOS_File(fileName)), @@ -23,7 +21,7 @@ Fast_FileInputStream::Fast_FileInputStream(const char *fileName) _fileOpenedOk = _theFile->OpenReadOnly(); } -Fast_FileInputStream::~Fast_FileInputStream(void) +Fast_FileInputStream::~Fast_FileInputStream() { Close(); delete _theFile; diff --git a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp index fddbea0fc41..7f5f6bed1eb 100644 --- a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp @@ -12,9 +12,8 @@ * ******************************************************************************/ -#include <vespa/fastos/fastos.h> -#include <vespa/fastlib/io/fileoutputstream.h> - +#include "fileoutputstream.h" +#include <vespa/fastos/file.h> Fast_FileOutputStream::Fast_FileOutputStream(const char *fileName) @@ -29,3 +28,15 @@ Fast_FileOutputStream::~Fast_FileOutputStream(void) _theFile->Close(); delete _theFile; } + +ssize_t +Fast_FileOutputStream::Write(const void *sourceBuffer, size_t bufferSize) { + return _theFile->CheckedWrite(sourceBuffer, bufferSize) ? + static_cast<ssize_t>(bufferSize) : + static_cast<ssize_t>(-1); +}; + +bool +Fast_FileOutputStream::Close() { + return _theFile->Close(); +}
\ No newline at end of file diff --git a/fastlib/src/vespa/fastlib/io/fileoutputstream.h b/fastlib/src/vespa/fastlib/io/fileoutputstream.h index eb97b3f8207..af7db086ed9 100644 --- a/fastlib/src/vespa/fastlib/io/fileoutputstream.h +++ b/fastlib/src/vespa/fastlib/io/fileoutputstream.h @@ -4,6 +4,8 @@ #include "outputstream.h" +class FastOS_FileInterface; + class Fast_FileOutputStream : public Fast_OutputStream { private: @@ -19,12 +21,8 @@ public: Fast_FileOutputStream(const char *fileName); ~Fast_FileOutputStream(); - ssize_t Write(const void *sourceBuffer, size_t bufferSize) override { - return _theFile->CheckedWrite(sourceBuffer, bufferSize) ? - static_cast<ssize_t>(bufferSize) : - static_cast<ssize_t>(-1); - }; + ssize_t Write(const void *sourceBuffer, size_t bufferSize) override; - bool Close() override { return _theFile->Close(); } + bool Close() override; void Flush() override {} }; diff --git a/fastlib/src/vespa/fastlib/io/outputstream.h b/fastlib/src/vespa/fastlib/io/outputstream.h index d09aa6e436a..ba4d74f1a7a 100644 --- a/fastlib/src/vespa/fastlib/io/outputstream.h +++ b/fastlib/src/vespa/fastlib/io/outputstream.h @@ -13,8 +13,7 @@ ******************************************************************************/ #pragma once - - +#include <cstdlib> class Fast_OutputStream { @@ -22,7 +21,7 @@ public: virtual ~Fast_OutputStream() { } - virtual bool Close(void) = 0; - virtual void Flush(void) = 0; + virtual bool Close() = 0; + virtual void Flush() = 0; virtual ssize_t Write(const void *sourceBuffer, size_t bufferSize) = 0; }; diff --git a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp b/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp index 81a31bd4b77..1a60a80dd7c 100644 --- a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp +++ b/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/testkit/test_kit.h> diff --git a/fastlib/src/vespa/fastlib/io/tests/bufferedstreamtest.cpp b/fastlib/src/vespa/fastlib/io/tests/bufferedstreamtest.cpp index b246d7afac6..e5f780c1ac0 100644 --- a/fastlib/src/vespa/fastlib/io/tests/bufferedstreamtest.cpp +++ b/fastlib/src/vespa/fastlib/io/tests/bufferedstreamtest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cassert> #include <ctime> #include <iostream> diff --git a/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp b/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp index e7d40ad9b75..e7386dcc3f4 100644 --- a/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp +++ b/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp @@ -12,9 +12,9 @@ * ******************************************************************************/ -#include <vespa/fastos/fastos.h> -#include <string.h> -#include <vespa/fastlib/net/httpchunkedinputstream.h> +#include "httpchunkedinputstream.h" +#include <cstring> +#include <cstdlib> Fast_HTTPChunkedInputStream::Fast_HTTPChunkedInputStream(Fast_InputStream &in) : Fast_FilterInputStream(in), _chunkSize(0), _inChunk(false), diff --git a/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp b/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp index 048ac85c8db..915d8e69223 100644 --- a/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp @@ -12,9 +12,10 @@ * ******************************************************************************/ -#include <vespa/fastos/fastos.h> -#include <string.h> -#include <vespa/fastlib/net/httpchunkedoutputstream.h> +#include "httpchunkedoutputstream.h" +#include <cassert> +#include <cstdio> +#include <cstring> Fast_HTTPChunkedOutputStream::Fast_HTTPChunkedOutputStream(Fast_OutputStream &out, size_t chunkSize) diff --git a/fastlib/src/vespa/fastlib/net/httpheaderparser.cpp b/fastlib/src/vespa/fastlib/net/httpheaderparser.cpp index 8f1dadef052..86fe3feabcf 100644 --- a/fastlib/src/vespa/fastlib/net/httpheaderparser.cpp +++ b/fastlib/src/vespa/fastlib/net/httpheaderparser.cpp @@ -1,7 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "httpheaderparser.h" #include <vespa/fastlib/io/bufferedinputstream.h> -#include <vespa/fastlib/net/httpheaderparser.h> +#include <cstdio> +#include <cstring> Fast_HTTPHeaderParser::Fast_HTTPHeaderParser(Fast_BufferedInputStream &in) : _pushBack(0), diff --git a/fastlib/src/vespa/fastlib/net/httpheaderparser.h b/fastlib/src/vespa/fastlib/net/httpheaderparser.h index ccd6c24ed37..94f6bccbdb4 100644 --- a/fastlib/src/vespa/fastlib/net/httpheaderparser.h +++ b/fastlib/src/vespa/fastlib/net/httpheaderparser.h @@ -2,6 +2,8 @@ #pragma once +#include <cstddef> + class Fast_BufferedInputStream; class Fast_HTTPHeaderParser diff --git a/fastlib/src/vespa/fastlib/net/socket.cpp b/fastlib/src/vespa/fastlib/net/socket.cpp index 1c7d915dc10..be463088bb3 100644 --- a/fastlib/src/vespa/fastlib/net/socket.cpp +++ b/fastlib/src/vespa/fastlib/net/socket.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "socket.h" Fast_Socket::~Fast_Socket() diff --git a/fastlib/src/vespa/fastlib/net/url.cpp b/fastlib/src/vespa/fastlib/net/url.cpp index 13cb0a5e2a7..a62a5171983 100644 --- a/fastlib/src/vespa/fastlib/net/url.cpp +++ b/fastlib/src/vespa/fastlib/net/url.cpp @@ -12,12 +12,9 @@ * */ -/** - * Encode and decode a URL - */ -#include <vespa/fastos/fastos.h> #include "url.h" +#include <cstdio> void Fast_URL::decode(const char *encodedURL, char *decodedURL, int bufsize) diff --git a/fastlib/src/vespa/fastlib/testsuite/cppttemplates.el b/fastlib/src/vespa/fastlib/testsuite/cppttemplates.el deleted file mode 100644 index a3eabb941d9..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/cppttemplates.el +++ /dev/null @@ -1,580 +0,0 @@ - -;; ------------------------------------------------------------------------ -;; Test class .h template -;; ------------------------------------------------------------------------ - -(defvar cppt-test-header-template "/** - * Definition of the automated unit test class for the CLASS_NAME class. - * - * @file FILE_NAME - * - * @author USER_NAME - * - * @date Created CREATION_DATE - * - * CVS_TAG - * - * <pre> - * Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * </pre> - ***************************************************************************/ -#ifndef TEST_CLASS_NAME_H -#define TEST_CLASS_NAME_H - -#include \"../CLASS_NAME.h\" -#include <map> -#include <fastlib/testsuite/test.h> -INCLUDE_CALLBACK - -/** - * The CLASS_NAMETest class holds - * the unit tests for the CLASS_NAME class. - * - * @sa CLASS_NAME - * @author USER_NAME - */ -class CLASS_NAMETest : public Test { - - /************************************************************************* - * Test methods - * - * This section contains boolean methods for testing each public method - * in the class ing tested - *************************************************************************/ - - /************************************************************************* - * Test administration methods - *************************************************************************/ - - /** - * Set up common stuff for all test methods. - * This method is called immediately before each test method is called - */ - bool setUp(); - - /** - * Tear down common stuff for all test methods. - * This method is called immediately after each test method is called - */ - void tearDown(); - - CALLBACK_TYPEDEF; - typedef std::map<std::string, tst_method_ptr> MethodContainer; - MethodContainer test_methods_; - void init(); - -protected: - - /** - * Since we are running within Emacs, the default behavior of - * print_progress which includes backspace does not work. - * We'll use a single '.' instead. - */ - virtual void print_progress() { *m_osptr << '.' << std::flush; } - -public: - - CLASS_NAMETest() : Test(\"CLASS_NAME\") { init(); } - ~CLASS_NAMETest() {} - - /************************************************************************* - * main entry points - *************************************************************************/ - void Run(MethodContainer::iterator &itr); - virtual void Run(); - void Run(const char *method); - void Run(int argc, char* argv[]); -}; - -#endif // TEST_CLASS_NAME_H - -// Local Variables: -// mode:c++ -// End: -" "This is the template for the header file for a single test class. -The 'CLASS_NAME' string is replaced with the name of the class to test. -Redefine this variable in your .emacs file, after (require 'cpptest) -if you want to change this template.") - - - -;; ------------------------------------------------------------------------ -;; Test class .cpp template -;; ------------------------------------------------------------------------ - -(defvar cppt-test-body-template "/** - * Implementation of the automated unit test class for the CLASS_NAME class. - * - * @file FILE_NAME - * - * @author USER_NAME - * - * @date Created CREATION_DATE - * - * CVS_TAG - * - * <pre> - * Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * </pre> - ***************************************************************************/ -#include <fastos/fastos.h> -#include \"TEST_HEADER.h\" -DEBUG_INCLUDES -EXTRA_INCLUDES - -/************************************************************************* - * Test methods - * - * This section contains boolean methods for testing each public method - * in the class being tested - *************************************************************************/ - -/************************************************************************* - * Test administration methods - *************************************************************************/ - -/** - * Set up common stuff for all test methods. - * This method is called immediately before each test method is called - */ -bool CLASS_NAMETest::setUp() { - return true; -} - -/** - * Tear down common stuff for all test methods. - * This method is called immediately after each test method is called - */ -void CLASS_NAMETest::tearDown() { -} - -/** - * Build up a map with all test methods - */ -void CLASS_NAMETest::init() { -} - -/************************************************************************* - * main entry points - *************************************************************************/ - - -void CLASS_NAMETest::Run(MethodContainer::iterator &itr) { - try { - DEBUG(\"CLASS_NAMETest\", 3, \"Running test method '\" + itr->first + \"'\"); - if (setUp()) { - CALL_CALLBACK; - tearDown(); - } - } catch (...) { - _fail(\"Got unknown exception in test method \" + itr->first); - } -} - -void CLASS_NAMETest::Run(const char* method) { - MethodContainer::iterator pos(test_methods_.find(method)); - if (pos != test_methods_.end()) { - Run(pos); - } else { - std::cerr << \"ERROR: No test method named \\\"\" - << method << \"\\\"\" << std::endl; - _fail(\"No such method\"); - } -} - -void CLASS_NAMETest::Run() { - for (MethodContainer::iterator itr(test_methods_.begin()); - itr != test_methods_.end(); - ++itr) - Run(itr); -} - -/* - * Parse runtime arguments before running. - * If the -m METHOD parameter is given, run only that method - */ -void CLASS_NAMETest::Run(int argc, char* argv[]) { - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], \"-m\") == 0 && argc > i + 1) { - Run(argv[++i]); - return; - } - } - Run(); -} -" "This is the template for the source file for a single test class. -The 'CLASS_NAME' string is replaced with the name of the class to test. -Redefine this variable in your .emacs file, after (require 'cpptest) -if you want to change this template.") - - - -(defvar cppt-notest-template " -// Comment out cerr below to ignore unimplemented tests -#define NOTEST(name) \\ -std::cerr << std::endl << __FILE__ << ':' << __LINE__ << \": \" \\ - << \"No test for method '\" << (name) << \"'\" << std::endl; -" "Definition of the NOTEST macro.") - - -;; ------------------------------------------------------------------------ -;; Test class application .cpp template -;; ------------------------------------------------------------------------ - -(defvar cppt-test-class-app-template "/** - * Definition and implementation of the application for running unit tests - * for the CLASS_NAME class in isolation. - * - * @file FILE_NAME - * - * @author USER_NAME - * - * @date Created CREATION_DATE - * - * CVS_TAG - * - * <pre> - * Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * </pre> - ****************************************************************************/ -#include <fastos/fastos.h> -#include \"TEST_HEADER.h\" -DEBUG_INCLUDES -EXTRA_INCLUDES - -/** - * The CLASS_NAMETestApp class is the main routine - * for running the unit tests for the CLASS_NAME class - * in isolation. - * - * @sa CLASS_NAME - * @author USER_NAME - */ -class CLASS_NAMETestApp : public FastOS_Application { -public: - virtual int Main() { - DEBUG_INIT - INIT_CODE - CLASS_NAMETest test; - test.SetStream(&std::cout); - test.Run(_argc, _argv); - return (int)test.Report(); - } -}; - -FASTOS_MAIN(CLASS_NAMETestApp); -" "This is the template for the test application for a single test class. -The 'CLASS_NAME' string is replaced with the name of the class to test. -Redefine this variable in your .emacs file, after (require 'cpptest) -if you want to change this template.") - -;; ------------------------------------------------------------------------ -;; Test suite template -;; ------------------------------------------------------------------------ - -(defvar cppt-suite-template "/** - * Implementation of the test suite application SUITE. - * - * @file FILE_NAME - * - * @author USER_NAME - * - * @date Created CREATION_DATE - * - * CVS_TAG - * - * <pre> - * Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * </pre> - ****************************************************************************/ -#include <fastos/fastos.h> -#include <fastlib/testsuite/suite.h> -DEBUG_INCLUDES -EXTRA_INCLUDES - -/** - * The SUITE class runs all the unit tests - * for the MODULE module. - * - * @author USER_NAME - */ -class SUITE : public Suite { - -public: - SUITE(); -}; - -SUITE::SUITE() : - Suite(\"SUITE\", &std::cout) -{ - // All tests for this module -} - -/** - * The SUITEApp class holds the main body for - * running the SUITE class. - * - * @author USER_NAME - */ -class SUITEApp : public FastOS_Application { -public: - virtual int Main(); -}; - -int SUITEApp::Main() { - DEBUG_INIT - INIT_CODE - SUITE suite; - suite.Run(); - long failures = suite.Report(); - suite.Free(); - return (int)failures; -} - -FASTOS_MAIN(SUITEApp); -" "This is the template for the source file for a complete test suite. -The 'SUITE' string is replaced with the directory of the classes to test, -with the string 'TestSute' appended. - -Redefine this variable in your .emacs file, after (require 'cpptest) -if you want to change this template.") - -;; ------------------------------------------------------------------------ -;; .cvsignore template -;; ------------------------------------------------------------------------ - -(defvar cppt-cvsignore-template - "*Test -*Suite -*test -*suite -dummylib -semantic.cache -*.log -Makefile -.depend" - "Default content for the .cvsignore file which is put into any newly -created test directories") - - - -;; ------------------------------------------------------------------------ -;; fastos.project template -;; ------------------------------------------------------------------------ - -(defvar cppt-project-template " -CUSTOMMAKE - -EXTRA_MAKE_TARGETS - -# Build any libraries given as dependencies in the cppt-pretest-target variable -%.a : ../*.h ../*.cpp - $(MAKE) -C $(@D) $(@F) - -# Don't warn for inline -# We need this for STL -ifeq ($(CC_PROG),gcc) -# Don't warn for inline -# We need this for STL -CFLAGS:=$(subst -Winline,,$(CFLAGS)) -Wno-ctor-dtor-privacy -# And force exceptions. -CXX_FLAGS := $(subst -fno-exceptions,-fexceptions,$(CXX_FLAGS)) -endif - -# Test the whole suite -.PHONY: test -test: _PRETEST_TARGET_ runSUITE - -# Run purify on the whole suite -purify: purifySUITE - -# Automatically rebuild .depend when source files change -.depend: $(wildcard *.h) $(wildcard *.cpp) - @$(ECHO_CMD) \"*** Generating new .depend...\" - @$(ECHO_CMD) > .depend.NEW $(DISCARDALL) - $(HIDE)$(CXX_MKDEP) $(INCLUDE_PATHS) $(SRCS_CPP) >> .depend.NEW - $(HIDE)$(MKDEP_POSTPROCESS) .depend.NEW > .depend - $(HIDE)$(DELETE_CMD) .depend.NEW - -ifeq ($(CC_PROG),cc) -# Don't optimize for better stacktrace in dbx. -CFLAGS := $(subst -xO3,,$(CFLAGS)) -endif - -# Use these for setting up and tearing down test scaffolding (test data etc) -PRE_TEST=_PRE_TEST_ -POST_TEST=_POST_TEST_ - -# Parameters to pass to the test programs -TEST_PARAM=CPPT_TEST_PARAMETERS -TEST_DB_FLAGS=CPPT_TEST_DBFLAGS - -################################################################### -# Unit test methods for single class -################################################################### - -# Run unit test for a single class -run%Test : %Test -\t$(HIDE)if [ x$(METHOD) = x ]; then \\ -\t \t$(ECHO_CMD) \"*** Testing class '$(subst Test,,$<)'\"; \\ -\t \t($(PRE_TEST) && \\ -\t \t ./$< $(TEST_PARAM) && \\ -\t \t $(POST_TEST)) || exit $$?; \\ -\t \t$(ECHO_CMD) \"*** All tests in class '$<' OK\"; \\ -\t else \\ -\t \t$(ECHO_CMD) \"*** Running test method '$(METHOD)' for class '$(subst Test,,$<)'\"; \\ -\t \t($(PRE_TEST) && \\ -\t \t ./$< -m $(METHOD) $(TEST_PARAM) && \\ -\t \t $(POST_TEST)) || exit $$?; \\ -\t \t$(ECHO_CMD) \"*** All tests in test method '$(subst METHOD=,,$(METHOD))' OK\"; \\ -\tfi - - -# Run unit test for a single class in debug mode -run%TestDebug : %Test -\t$(HIDE)if [ x$(METHOD) = x ]; then \\ -\t \t$(ECHO_CMD) \"*** Testing class '$(subst TestDebug,,$<)' in debug mode\"; \\ -\t \t($(PRE_TEST) && \\ -\t \t ./$< $(TEST_PARAM) $(TEST_DB_FLAGS) && \\ -\t \t $(POST_TEST)) || exit $$?; \\ -\t \t$(ECHO_CMD) \"*** All tests in class '$<' OK\"; \\ -\t else \\ -\t \t$(ECHO_CMD) \"*** Running test method '$(METHOD)' for class '$(subst TestDebug,,$<)' in debug mode\"; \\ -\t \t($(PRE_TEST) && \\ -\t \t ./$< -m $(METHOD) $(TEST_PARAM) $(TEST_DB_FLAGS) && \\ -\t \t $(POST_TEST)) || exit $$?; \\ -\t \t$(ECHO_CMD) \"*** All tests in test method '$(subst METHOD=,,$(METHOD))' OK\"; \\ -\tfi - -# Run unit test for a single class in purify -purify%Test : Purified%Test -\t$(HIDE)$(ECHO_CMD) \"*** Testing class with Purify\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK\" - -# Run unit test for a single class in purify with debugging -purify%TestDebug : Purified%Test -\t$(HIDE)$(ECHO_CMD) \"*** Testing class with Purify in debug mode\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) $(TEST_DB_FLAGS) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK\" - -################################################################### -# Unit test methods for test suite -################################################################### - -# Run unit tests for all classes -run%TestSuite : %TestSuite -\t$(HIDE)$(ECHO_CMD) \"*** Testing suite '$(subst TestSuite,,$<)'\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK\" - -# Run unit tests for all classes in the package in debug mode -run%TestSuiteDebug : %TestSuite -\t$(HIDE)$(ECHO_CMD) \"*** Testing suite '$(subst TestSuiteDebug,,$<)' in debug mode\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) $(TEST_DB_FLAGS) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK\" - -# Run unit tests for all classes in purify mode -purify%TestSuite : Purified%TestSuite -\t$(HIDE)$(ECHO_CMD) \"*** Testing suite with Purify\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK **\" - -# Run unit tests for all classes in purify mode with debugging -purify%TestSuiteDebug : Purified%TestSuite -\t$(HIDE)$(ECHO_CMD) \"*** Testing suite with Purify mode in debug mode\" -\t$(HIDE)($(PRE_TEST) && \\ -\t ./$< $(TEST_PARAM) $(TEST_DB_FLAGS) && \\ -\t $(POST_TEST)) || exit $$? -\t$(HIDE)$(ECHO_CMD) \"*** All tests in $< OK **\" - -################################################################ -# Build purify executables -################################################################ -Purified%: % -\t$(HIDE)$(ECHO_CMD) \"*** Building purify binary $@\" -\t$(HIDE)purify -chain-length=12 -max-threads=250 \ -\t-static-checking-default=aggressive -always-use-cache-dir=yes \ -\tCC $(CXX_FLAGS) $(LOCALFLAGS) \ -\t$(APP_$(shell echo $< | tr '[:lower:]' '[:upper:]')_OBJS) \ -\t$(COMPILEEXEOUTPUT_FLAG) $@ \ -\t$($(MAKETARGET)LIB_APP_$(shell echo $< | tr '[:lower:]' '[:upper:]')) \ -\t$(LINK_FLAGS) -\t$(HIDE)$(ECHO_CMD) \"*** Done $<\" -" - "This is the template for an empty fastos.project file. - -The 'SUITE' string is replaced with the directory of the classes to test, -with the string 'TestSute' appended. - -The 'EXTRA_MAKE_TARGETS' string is replaced with the value of the -cppt-extra-make-targets variable - -Redefine this variable in your .emacs file, after (require 'cpptest) -if you want to change this template.") - -;; ----------------------------------------------------------------------- -;; Documentation templates -;; ----------------------------------------------------------------------- - -(defvar cppt-file-doc-template "/** - * - * @file FILE_NAME - * @author USER_NAME - * @date Created CREATION_DATE - * CVS_TAG - * - * <pre> - * Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * </pre> - ****************************************************************************/ -" - "This is a template for generating a file documenation.The USER_NAME -string will be replaced by the full name of the user. The -CREATION_DATE string will be replaced by todays date. The YEAR string -will be replaced by the current year. The FILE_NAME string will be -replaced by the name of the file.") - -(defvar cppt-class-doc-template "/** - * The CLASS_NAME TYPE - * - * @sa anotherClass::anotherMethod - * @author USER_NAME - */ -" - "This is a template for generating a default class documenation. The -TYPE string will be replaced by class or namsespace. The CLASS_NAME_NAME -string will be replaced by the name of the class or namespace. The -USER_NAME string will be replaced by the full name of the user.") - -(defvar cppt-method-doc-template "/** - * The METHOD_NAME - * - * @author USER_NAME -RETURN_TYPE -EXCEPTIONS -PARAMETERS - */ -" - "This is a template for generating a default method -documenation. The METHOD_NAME string will be replaced by the name of -the method. The USER_NAME string will be replaced by the fulll name of -the user. The RETURN_TYPE string will be replaced by the return type -of the method (if not a ctor or dtor). The PARAMETERS string will be -replaced by a series of '* @parameter PARAMETER_NAME a PARAMETER_TYPE -value' strings for each parameter of the method, if any.") - - - - -(provide 'cppttemplates)
\ No newline at end of file diff --git a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp b/fastlib/src/vespa/fastlib/text/apps/extcase.cpp index 1200465331c..5f7579287e0 100644 --- a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp +++ b/fastlib/src/vespa/fastlib/text/apps/extcase.cpp @@ -14,7 +14,6 @@ * ALL RIGHTS RESERVED */ -#include <vespa/fastos/fastos.h> #include <vespa/fastlib/io/bufferedfile.h> unsigned int lowercase[65536]; diff --git a/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp b/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp index a0ee05b9b5d..8a1260a5355 100644 --- a/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp +++ b/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/fastlib/text/unicodeutil.h> +#include <vespa/fastos/app.h> class UnicodePropertyDumpApp : public FastOS_Application { diff --git a/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp b/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp index ffea4d735ba..b000645295a 100644 --- a/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp +++ b/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/fastlib/text/unicodeutil.h> +#include <vespa/fastos/app.h> class UnicodeToLowerDumpApp : public FastOS_Application { diff --git a/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp b/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp index fed5ab14e5f..e216e5034d6 100644 --- a/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp +++ b/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "characterclasstest.h" int character_class_test_app::Main() diff --git a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp b/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp index 6b812ccd74d..69c0384afe5 100644 --- a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp +++ b/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "latintokenizertest.h" int LatinTokenizerTestApp::Main() diff --git a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h b/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h index 88275199bca..4115e3f7a05 100644 --- a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h +++ b/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h @@ -2,9 +2,9 @@ #pragma once #include <vespa/fastlib/testsuite/test.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/fastlib/text/latintokenizer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fastos/app.h> class Mapel_Pucntuation { private: diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp index c4372f2e3ac..003f4ee7c5b 100644 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp +++ b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wordfolderstest.h" int WordFoldersTestApp::Main() diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h index 3224426f51d..eb717ad489d 100644 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h +++ b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastlib/testsuite/test.h> #include <vespa/fastlib/text/normwordfolder.h> +#include <vespa/fastos/app.h> +#include <memory> +#include <cstring> class WordFoldersTest : public Test { diff --git a/fastlib/src/vespa/fastlib/text/unicodeutil.cpp b/fastlib/src/vespa/fastlib/text/unicodeutil.cpp index 822f8044ded..c0c88a61728 100644 --- a/fastlib/src/vespa/fastlib/text/unicodeutil.cpp +++ b/fastlib/src/vespa/fastlib/text/unicodeutil.cpp @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/fastlib/text/unicodeutil.h> +#include "unicodeutil.h" #include <cstdlib> +#include <cstdint> +#include <cassert> #include "unicodeutil-charprops.cpp" #include "unicodeutil-lowercase.cpp" @@ -22,7 +23,7 @@ Initialize _G_Initializer; } void -Fast_UnicodeUtil::InitTables(void) +Fast_UnicodeUtil::InitTables() { /** * Hack for Katakana accent marks (torgeir) diff --git a/fastlib/src/vespa/fastlib/text/wordfolder.cpp b/fastlib/src/vespa/fastlib/text/wordfolder.cpp index de4f0daae3e..221960e046c 100644 --- a/fastlib/src/vespa/fastlib/text/wordfolder.cpp +++ b/fastlib/src/vespa/fastlib/text/wordfolder.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/fastlib/text/wordfolder.h> -Fast_WordFolder::~Fast_WordFolder(void) +#include "wordfolder.h" + +Fast_WordFolder::~Fast_WordFolder() { } diff --git a/fastlib/src/vespa/fastlib/util/bag.h b/fastlib/src/vespa/fastlib/util/bag.h index af90571c5fc..c703e10e395 100644 --- a/fastlib/src/vespa/fastlib/util/bag.h +++ b/fastlib/src/vespa/fastlib/util/bag.h @@ -264,8 +264,8 @@ public: * Fast_Bag */ Fast_BagIterator(void) : - _bag(NULL), - _array(NULL), + _bag(nullptr), + _array(nullptr), _index(0), _end(true) { @@ -412,7 +412,7 @@ Fast_Bag<Type>::Fast_Bag() : template <class Type> Fast_Bag<Type>::Fast_Bag(const Fast_Bag<Type>& source) : _capacity(source._capacity), - _array(NULL), + _array(nullptr), _blocksize(source._blocksize), _numElements(source._numElements) { @@ -462,7 +462,7 @@ inline Fast_Bag<Type>& Fast_Bag<Type>::operator=(const Fast_Bag<Type>& other) // Self assignment if(this == &other) return *this; - if(_array != NULL) { + if(_array != nullptr) { delete[] _array; } diff --git a/fastlib/src/vespa/fastlib/util/base64.cpp b/fastlib/src/vespa/fastlib/util/base64.cpp index c87cba28e5d..7cd9b35edac 100644 --- a/fastlib/src/vespa/fastlib/util/base64.cpp +++ b/fastlib/src/vespa/fastlib/util/base64.cpp @@ -6,17 +6,9 @@ * Utility functions for base-64 encoding/decoding. *****************************************************************************/ -#include <vespa/fastos/fastos.h> -#include <stdio.h> -#include <ctype.h> #include "base64.h" - -/** -***************************************************************************** -* Define lookup table. -* -* @author Aleksander Øhrn -*****************************************************************************/ +#include <cstdio> +#include <cctype> static const char _base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/fastlib/src/vespa/fastlib/util/tests/bagtest.cpp b/fastlib/src/vespa/fastlib/util/tests/bagtest.cpp index b06319c238a..a547ac98507 100644 --- a/fastlib/src/vespa/fastlib/util/tests/bagtest.cpp +++ b/fastlib/src/vespa/fastlib/util/tests/bagtest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bagtest.h" diff --git a/fastlib/src/vespa/fastlib/util/tests/bagtest.h b/fastlib/src/vespa/fastlib/util/tests/bagtest.h index 7da461c0189..daf0c2d507a 100644 --- a/fastlib/src/vespa/fastlib/util/tests/bagtest.h +++ b/fastlib/src/vespa/fastlib/util/tests/bagtest.h @@ -2,6 +2,7 @@ #include <vespa/fastlib/util/bag.h> #include <vespa/fastlib/testsuite/test.h> +#include <vespa/fastos/app.h> #include <iostream> /** diff --git a/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp b/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp index a4d8320a29c..7121a9c83af 100644 --- a/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp +++ b/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <iostream> #include "../wildcard_match.h" diff --git a/fastos/src/tests/coretest.cpp b/fastos/src/tests/coretest.cpp index a9efcc7090b..712bdf2cfd9 100644 --- a/fastos/src/tests/coretest.cpp +++ b/fastos/src/tests/coretest.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> static void diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp index bc8fc0bef0c..a4248095b9b 100644 --- a/fastos/src/tests/coretest2.cpp +++ b/fastos/src/tests/coretest2.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> static void diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp index 3865920f981..2d0f5c6e415 100644 --- a/fastos/src/tests/performancetest.cpp +++ b/fastos/src/tests/performancetest.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <stdlib.h> -#include <vespa/fastos/fastos.h> #include "tests.h" void PerformanceTest (char *buffer); diff --git a/fastos/src/tests/processtest.cpp b/fastos/src/tests/processtest.cpp index a4b1e38b03b..d6a0ac169aa 100644 --- a/fastos/src/tests/processtest.cpp +++ b/fastos/src/tests/processtest.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tests.h" +#include <vespa/fastos/process.h> +#include <vespa/fastos/time.h> +#include <vespa/fastos/file.h> class MyListener : public FastOS_ProcessRedirectListener { diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h index cc54e66292f..6807741f86f 100644 --- a/fastos/src/tests/tests.h +++ b/fastos/src/tests/tests.h @@ -2,6 +2,7 @@ #include <vespa/fastos/app.h> #include <vespa/fastos/socket.h> +#include <vespa/fastos/thread.h> class BaseTest : public FastOS_Application { diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp index 60a05f67151..dea5d5016ec 100644 --- a/fastos/src/tests/typetest.cpp +++ b/fastos/src/tests/typetest.cpp @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> -#include <vespa/fastos/fastos.h> #include "tests.h" +#include <vespa/fastos/file.h> +#include <vespa/fastos/serversocket.h> +#include <cstdlib> class TypeTest : public BaseTest { diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h index 75fb5c30d43..faa6ff55c7d 100644 --- a/fastos/src/vespa/fastos/app.h +++ b/fastos/src/vespa/fastos/app.h @@ -13,8 +13,8 @@ #include <vespa/fastos/types.h> class FastOS_ProcessInterface; +class FastOS_ThreadPool; -#include <vespa/fastos/thread.h> #include <vespa/fastos/mutex.h> /** diff --git a/fastos/src/vespa/fastos/fastos.h b/fastos/src/vespa/fastos/fastos.h deleted file mode 100644 index dd03a3c512c..00000000000 --- a/fastos/src/vespa/fastos/fastos.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * @file - * Main include file for FastOS. This pulls in all the neccessary - * definitions. - * - * @author Div, Oivind H. Danielsen - */ - - -#pragma once - -#include <vespa/fastos/types.h> -#include <vespa/fastos/app.h> -#include <vespa/fastos/file.h> -#include <vespa/fastos/mutex.h> -#include <vespa/fastos/cond.h> -#include <vespa/fastos/thread.h> -#include <vespa/fastos/socket.h> -#include <vespa/fastos/serversocket.h> -#include <vespa/fastos/timestamp.h> -#include <vespa/fastos/time.h> -#include <vespa/fastos/prefetch.h> -#include <vespa/fastos/process.h> -#include <vespa/fastos/dynamiclibrary.h> - diff --git a/fastos/src/vespa/fastos/time.h b/fastos/src/vespa/fastos/time.h index 97ebe21a3b5..1e37813a072 100644 --- a/fastos/src/vespa/fastos/time.h +++ b/fastos/src/vespa/fastos/time.h @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> - /** * Interface to OS time functions. */ @@ -12,7 +10,7 @@ protected: /** * Destructor. No cleanup needed for base class. */ - virtual ~FastOS_TimeInterface(void) { } + virtual ~FastOS_TimeInterface() { } public: /** @@ -49,30 +47,26 @@ public: * Note: Only millisecond accuracy is guaranteed. * @param microsecs Number of microseconds to add. */ - void AddMicroSecs(double microsecs) - { SetMicroSecs(MicroSecs() + microsecs); } + void AddMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() + microsecs); } /** * Add a specified number of milliseconds to the time. * @param millisecs Number of milliseconds to add. */ - void AddMilliSecs(double millisecs) - { SetMilliSecs(MilliSecs() + millisecs); } + void AddMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() + millisecs); } /** * Subtract a specified number of microseconds from the time. * Note: Only millisecond accuracy is guaranteed. * @param microsecs Number of microseconds to subtract. */ - void SubtractMicroSecs(double microsecs) - { SetMicroSecs(MicroSecs() - microsecs); } + void SubtractMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() - microsecs); } /** * Subtract a specified number of milliseconds from the time. * @param millisecs Number of milliseconds to subtract. */ - void SubtractMilliSecs(double millisecs) - { SetMilliSecs(MilliSecs() - millisecs); } + void SubtractMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() - millisecs); } /** * Return the time in microseconds. @@ -133,7 +127,6 @@ public: virtual long int GetMicroSeconds() const = 0; }; - #include <vespa/fastos/unix_time.h> -typedef FastOS_UNIX_Time FASTOS_PREFIX(Time); +using FastOS_Time = FastOS_UNIX_Time; diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h index d3b3f0f3d96..7438b96d434 100644 --- a/fastos/src/vespa/fastos/unix_process.h +++ b/fastos/src/vespa/fastos/unix_process.h @@ -13,7 +13,7 @@ #include <string> #include <memory> -class FastOS_ThreadPool; +class FastOS_BoolCond; class FastOS_UNIX_RealProcess; #include <vespa/fastos/ringbuffer.h> diff --git a/fastos/src/vespa/fastos/unix_time.cpp b/fastos/src/vespa/fastos/unix_time.cpp index 202ffe6d82a..6b6a8e58ffb 100644 --- a/fastos/src/vespa/fastos/unix_time.cpp +++ b/fastos/src/vespa/fastos/unix_time.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/time.h> +#include "time.h" +#include "types.h" double FastOS_UNIX_Time::MicroSecs() const @@ -96,3 +97,8 @@ FastOS_UNIX_Time::SetSecs(double secs) _time.tv_usec = - static_cast<int>((- secs - (-_time.tv_sec)) * 1000000); } } + +void FastOS_UNIX_Time::SetNow() { + gettimeofday(&_time, NULL); +} + diff --git a/fastos/src/vespa/fastos/unix_time.h b/fastos/src/vespa/fastos/unix_time.h index dd6122bc463..5ae4916418a 100644 --- a/fastos/src/vespa/fastos/unix_time.h +++ b/fastos/src/vespa/fastos/unix_time.h @@ -89,9 +89,8 @@ public: void SetMilliSecs(double millisecs) override; void SetSecs(double secs) override; - void SetNow() override { gettimeofday(&_time, NULL); } + void SetNow() override; long int GetSeconds() const override { return _time.tv_sec; } long int GetMicroSeconds() const override { return _time.tv_usec; } }; - diff --git a/fbench/src/fbench/client.cpp b/fbench/src/fbench/client.cpp index f539e990ad9..94d0a78631b 100644 --- a/fbench/src/fbench/client.cpp +++ b/fbench/src/fbench/client.cpp @@ -34,6 +34,120 @@ void Client::runMe(Client * me) { me->run(); } + +class UrlReader { + FileReader &_reader; + const ClientArguments &_args; + int _restarts; + int _contentbufsize; + int _leftOversLen; + char *_contentbuf; + const char *_leftOvers; +public: + UrlReader(FileReader& reader, const ClientArguments &args) + : _reader(reader), _args(args), _restarts(0), + _contentbufsize(0), _leftOversLen(0), + _contentbuf(0), _leftOvers(0) + { + if (_args._usePostMode) { + _contentbufsize = 16 * _args._maxLineSize; + _contentbuf = new char[_contentbufsize]; + } + } + bool reset(); + int findUrl(char *buf, int buflen); + int nextUrl(char *buf, int buflen); + int nextContent(); + const char *content() const { return _contentbuf; } + ~UrlReader() { delete [] _contentbuf; } +}; + +bool UrlReader::reset() +{ + if (_restarts == _args._restartLimit) { + return false; + } else if (_args._restartLimit > 0) { + _restarts++; + } + _reader.Reset(); + // Start reading from offset + if (_args._singleQueryFile) { + _reader.SetFilePos(_args._queryfileOffset); + } + return true; +} + +int UrlReader::findUrl(char *buf, int buflen) +{ + while (true) { + if ( _args._singleQueryFile && _reader.GetFilePos() >= _args._queryfileEndOffset ) { + // reached logical EOF + return -1; + } + int ll = _reader.ReadLine(buf, buflen); + if (ll < 0) { + // reached physical EOF + return ll; + } + if (ll > 0) { + if (buf[0] == '/' || !_args._usePostMode) { + // found URL + return ll; + } + } + } +} + +int UrlReader::nextUrl(char *buf, int buflen) +{ + if (_leftOvers) { + int sz = std::min(_leftOversLen, buflen-1); + strncpy(buf, _leftOvers, sz); + buf[sz] = '\0'; + _leftOvers = NULL; + return _leftOversLen; + } + int ll = findUrl(buf, buflen); + if (ll > 0) { + return ll; + } + if (reset()) { + // try again + ll = findUrl(buf, buflen); + } + return ll; +} + +int UrlReader::nextContent() +{ + char *buf = _contentbuf; + int totLen = 0; + // make sure we don't chop leftover URL + while (totLen + _args._maxLineSize < _contentbufsize) { + // allow space for newline: + int room = _contentbufsize - totLen - 1; + int len = _reader.ReadLine(buf, room); + if (len < 0) { + // reached EOF + break; + } + len = std::min(len, room); + if (len > 0 && buf[0] == '/') { + // reached next URL + _leftOvers = buf; + _leftOversLen = len; + break; + } + buf += len; + totLen += len; + *buf++ = '\n'; + totLen++; + } + // ignore last newline + return (totLen > 0) ? totLen-1 : 0; +} + + void Client::run() { @@ -73,37 +187,24 @@ Client::run() if ( _args->_singleQueryFile ) _reader->SetFilePos(_args->_queryfileOffset); + UrlReader urlSource(*_reader, *_args); + size_t urlNumber = 0; + // run queries while (!_stop) { _cycleTimer->Start(); - linelen = _reader->ReadLine(_linebuf, _linebufsize); - - // Read maximum to _queryfileOffsetEnd - if ( _args->_singleQueryFile && _reader->GetBufPos() >= _args->_queryfileBytes ) { - _reader->SetFilePos(_args->_queryfileOffset); - } - - if (linelen < 0) { - _reader->Reset(); - // Start reading from offset - if ( _args->_singleQueryFile ) { - _reader->SetFilePos(_args->_queryfileOffset); - } - - linelen = _reader->ReadLine(_linebuf, _linebufsize); - if (linelen < 0) { + linelen = urlSource.nextUrl(_linebuf, _linebufsize); + if (linelen > 0) { + ++urlNumber; + } else { + if (urlNumber == 0) { fprintf(stderr, "Client %d: ERROR: could not read any lines from '%s'\n", _args->_myNum, inputFilename); _status->SetError("Could not read any lines from query file."); - break; - } - if (_args->_restartLimit == 0) { - break; - } else if (_args->_restartLimit > 0) { - _args->_restartLimit--; } + break; } if (linelen < _linebufsize) { if (_output) { @@ -114,8 +215,9 @@ Client::run() if (linelen + (int)_args->_queryStringToAppend.length() < _linebufsize) { strcat(_linebuf, _args->_queryStringToAppend.c_str()); } + int cLen = _args->_usePostMode ? urlSource.nextContent() : 0; _reqTimer->Start(); - auto fetch_status = _http->Fetch(_linebuf, _output.get()); + auto fetch_status = _http->Fetch(_linebuf, _output.get(), _args->_usePostMode, urlSource.content(), cLen); _reqTimer->Stop(); _status->AddRequestStatus(fetch_status.RequestStatus()); if (fetch_status.Ok() && fetch_status.TotalHitCount() == 0) diff --git a/fbench/src/fbench/client.h b/fbench/src/fbench/client.h index 4e78d2d6adc..052978766ff 100644 --- a/fbench/src/fbench/client.h +++ b/fbench/src/fbench/client.h @@ -95,13 +95,16 @@ struct ClientArguments **/ bool _keepAlive; + /** Whether we should use POST in requests */ + bool _usePostMode; + /** * Indicate whether to add benchmark data coverage headers **/ bool _headerBenchmarkdataCoverage; uint64_t _queryfileOffset; - uint64_t _queryfileBytes; + uint64_t _queryfileEndOffset; bool _singleQueryFile; std::string _queryStringToAppend; std::string _extraHeaders; @@ -115,9 +118,9 @@ struct ClientArguments int ignoreCount, int byteLimit, int restartLimit, int maxLineSize, bool keepAlive, bool headerBenchmarkdataCoverage, - uint64_t queryfileOffset, uint64_t queryfileBytes, bool singleQueryFile, + uint64_t queryfileOffset, uint64_t queryfileEndOffset, bool singleQueryFile, const std::string & queryStringToAppend, const std::string & extraHeaders, - const std::string &authority) + const std::string &authority, bool postMode) : _myNum(myNum), _totNum(totNum), _filenamePattern(filenamePattern), @@ -131,9 +134,10 @@ struct ClientArguments _restartLimit(restartLimit), _maxLineSize(maxLineSize), _keepAlive(keepAlive), + _usePostMode(postMode), _headerBenchmarkdataCoverage(headerBenchmarkdataCoverage), _queryfileOffset(queryfileOffset), - _queryfileBytes(queryfileBytes), + _queryfileEndOffset(queryfileEndOffset), _singleQueryFile(singleQueryFile), _queryStringToAppend(queryStringToAppend), _extraHeaders(extraHeaders), diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 14a0fd4106d..20195d9d234 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -20,6 +20,7 @@ FBench::FBench() _restartLimit(0), _maxLineSize(0), _keepAlive(true), + _usePostMode(false), _headerBenchmarkdataCoverage(false), _seconds(60), _singleQueryFile(false) @@ -39,7 +40,7 @@ FBench::InitBenchmark(int numClients, int ignoreCount, int cycle, int byteLimit, int restartLimit, int maxLineSize, bool keepAlive, bool headerBenchmarkdataCoverage, int seconds, bool singleQueryFile, const std::string & queryStringToAppend, const std::string & extraHeaders, - const std::string &authority) + const std::string &authority, bool postMode) { _clients.resize(numClients); _ignoreCount = ignoreCount; @@ -57,6 +58,7 @@ FBench::InitBenchmark(int numClients, int ignoreCount, int cycle, _restartLimit = restartLimit; _maxLineSize = maxLineSize; _keepAlive = keepAlive; + _usePostMode = postMode; _headerBenchmarkdataCoverage = headerBenchmarkdataCoverage; _seconds = seconds; _singleQueryFile = singleQueryFile; @@ -69,6 +71,12 @@ FBench::CreateClients() int i(0); for(auto & client : _clients) { + uint64_t off_beg = 0; + uint64_t off_end = 0; + if (_singleQueryFile) { + off_beg = _queryfileOffset[i]; + off_end = _queryfileOffset[i+1]; + } client = std::make_unique<Client>( new ClientArguments(i, _clients.size(), _filenamePattern, _outputPattern, _hostnames[i % _hostnames.size()].c_str(), @@ -76,9 +84,8 @@ FBench::CreateClients() random() % spread, _ignoreCount, _byteLimit, _restartLimit, _maxLineSize, _keepAlive, _headerBenchmarkdataCoverage, - _queryfileOffset[i % _queryfileOffset.size()], - _queryfileOffset[i+1 % _queryfileOffset.size()]-_queryfileOffset[i % _queryfileOffset.size()], - _singleQueryFile, _queryStringToAppend, _extraHeaders, _authority)); + off_beg, off_end, + _singleQueryFile, _queryStringToAppend, _extraHeaders, _authority, _usePostMode)); ++i; } } @@ -205,6 +212,7 @@ FBench::Usage() printf(" [-r restartLimit] [-m maxLineSize] [-k] <hostname> <port>\n\n"); printf(" -H <str> : append extra header to each get request.\n"); printf(" -A <str> : assign autority. <str> should be hostname:port format. Overrides Host: header sent.\n"); + printf(" -P : use POST for requests instead of GET.\n"); printf(" -a <str> : append string to each query\n"); printf(" -n <num> : run with <num> parallel clients [10]\n"); printf(" -c <num> : each client will make a request each <num> milliseconds [1000]\n"); @@ -258,6 +266,7 @@ FBench::Main(int argc, char *argv[]) int restartLimit = -1; bool keepAlive = true; bool headerBenchmarkdataCoverage = false; + bool usePostMode = false; bool singleQueryFile = false; std::string authority; @@ -272,7 +281,7 @@ FBench::Main(int argc, char *argv[]) idx = 1; optError = false; - while((opt = GetOpt(argc, argv, "H:A:a:n:c:l:i:s:q:o:r:m:p:kxyz", arg, idx)) != -1) { + while((opt = GetOpt(argc, argv, "H:A:a:n:c:l:i:s:q:o:r:m:p:kxyzP", arg, idx)) != -1) { switch(opt) { case 'A': authority = arg; @@ -317,6 +326,9 @@ FBench::Main(int argc, char *argv[]) maxLineSize = minLineSize; } break; + case 'P': + usePostMode = true; + break; case 'p': printInterval = atoi(arg); if (printInterval < 0) @@ -388,12 +400,12 @@ FBench::Main(int argc, char *argv[]) FileReader r; r.Open(queryFilePattern); uint64_t clientOffset = std::max(i*perClient, _queryfileOffset.back() ); - uint64_t newline = r.FindNewline(clientOffset) + 1; + uint64_t newline = r.FindNextLine(clientOffset); _queryfileOffset.push_back(newline); } // Add pos to end of file - _queryfileOffset.push_back( totalSize+1 ); + _queryfileOffset.push_back(totalSize); // Print offset of clients @@ -410,7 +422,7 @@ FBench::Main(int argc, char *argv[]) keepAlive, headerBenchmarkdataCoverage, seconds, singleQueryFile, queryStringToAppend, extraHeaders, - authority); + authority, usePostMode); CreateClients(); StartClients(); diff --git a/fbench/src/fbench/fbench.h b/fbench/src/fbench/fbench.h index 636d2b7e8b7..dcf56ce6336 100644 --- a/fbench/src/fbench/fbench.h +++ b/fbench/src/fbench/fbench.h @@ -22,6 +22,7 @@ private: int _restartLimit; int _maxLineSize; bool _keepAlive; + bool _usePostMode; bool _headerBenchmarkdataCoverage; int _seconds; std::vector<uint64_t> _queryfileOffset; @@ -36,7 +37,7 @@ private: int byteLimit, int restartLimit, int maxLineSize, bool keepAlive, bool headerBenchmarkdataCoverage, int seconds, bool singleQueryFile, const std::string & queryStringToAppend, const std::string & extraHeaders, - const std::string &authority); + const std::string &authority, bool postMode); void CreateClients(); void StartClients(); diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index ce2157335e4..8974435a030 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -94,7 +94,7 @@ HTTPClient::ReadLine(char *buf, size_t bufsize) } bool -HTTPClient::Connect(const char *url) +HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen) { char tmp[4096]; char *req = NULL; @@ -121,24 +121,24 @@ HTTPClient::Connect(const char *url) assert(req != NULL); } - if (headers.length() > 0) { - headers += "\r\n"; + if (!_keepAlive) { + headers += "Connection: close\r\n"; } + headers += "User-Agent: fbench/4.2.10\r\n"; + // create request - if(_keepAlive) { + if (usePost) { snprintf(req, req_max, - "GET %s HTTP/1.1\r\n" + "POST %s HTTP/1.1\r\n" "Host: %s\r\n" - "User-Agent: fbench/4.2.10\r\n" + "Content-Length: %d\r\n" "%s" "\r\n", - url, _authority.c_str(), headers.c_str()); + url, _authority.c_str(), cLen, headers.c_str()); } else { snprintf(req, req_max, "GET %s HTTP/1.1\r\n" "Host: %s\r\n" - "Connection: close\r\n" - "User-Agent: fbench/4.2.10\r\n" "%s" "\r\n", url, _authority.c_str(), headers.c_str()); @@ -148,6 +148,7 @@ HTTPClient::Connect(const char *url) if (_keepAlive && _socket->IsOpened() && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen) && FillBuffer() > 0) { // DEBUG @@ -167,7 +168,9 @@ HTTPClient::Connect(const char *url) && _socket->Connect() && _socket->SetNoDelay(true) && _socket->SetSoLinger(false, 0) - && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req)) { + && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen)) + { // DEBUG // printf("New Socket connection!\n"); @@ -187,6 +190,7 @@ HTTPClient::Connect(const char *url) return false; } + char * HTTPClient::SplitString(char *input, int &argc, char **argv, int maxargs) { @@ -338,7 +342,7 @@ HTTPClient::ReadChunkHeader() } bool -HTTPClient::Open(const char *url) +HTTPClient::Open(const char *url, bool usePost, const char *content, int cLen) { if (_isOpen) Close(); @@ -346,7 +350,7 @@ HTTPClient::Open(const char *url) ResetBuffer(); _dataRead = 0; _dataDone = false; - _isOpen = Connect(url); + _isOpen = Connect(url, usePost, content, cLen); if(!_isOpen || !ReadHTTPHeader()) { Close(); return false; @@ -508,14 +512,15 @@ HTTPClient::Close() } HTTPClient::FetchStatus -HTTPClient::Fetch(const char *url, std::ostream *file) +HTTPClient::Fetch(const char *url, std::ostream *file, + bool usePost, const char *content, int contentLen) { size_t buflen = FETCH_BUFLEN; char buf[FETCH_BUFLEN]; // NB: ensure big enough thread stack. ssize_t readRes = 0; ssize_t written = 0; - if (!Open(url)) { + if (!Open(url, usePost, content, contentLen)) { return FetchStatus(false, _requestStatus, _totalHitCount, 0); } diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h index 831ff83651d..d7e7d45424d 100644 --- a/fbench/src/httpclient/httpclient.h +++ b/fbench/src/httpclient/httpclient.h @@ -157,7 +157,8 @@ protected: * @return success(true)/failure(false) * @param url the url you want to connect to **/ - bool Connect(const char *url); + bool Connect(const char *url, bool usePost = false, + const char *content = NULL, int contentLen = 0); /** * Read the next line of text from the data stream into 'buf'. If @@ -243,8 +244,11 @@ public: * * @return success(true)/failure(false) * @param url the url you want to connect to + * @param usePost whether to use POST in the request + * @param content if usePost is true, the content to post + * @param cLen length of content in bytes **/ - bool Open(const char *url); + bool Open(const char *url, bool usePost = false, const char *content = 0, int cLen = 0); /** * Read data from the url we are currently connected to. This method @@ -326,7 +330,10 @@ public: * @param url the url to fetch. * @param file where to save the fetched document. If this parameter * is NULL, the content will be read and then discarded. + * @param usePost whether to use POST in the request + * @param content if usePost is true, the content to post + * @param contentLen length of content in bytes **/ - FetchStatus Fetch(const char *url, std::ostream *file = NULL); + FetchStatus Fetch(const char *url, std::ostream *file = NULL, + bool usePost = false, const char *content = NULL, int contentLen = 0); }; - diff --git a/fbench/src/util/filereader.cpp b/fbench/src/util/filereader.cpp index b1eebbcb2f0..e59e9966899 100644 --- a/fbench/src/util/filereader.cpp +++ b/fbench/src/util/filereader.cpp @@ -20,6 +20,8 @@ FileReader::FileReader() _file(&std::cin), _bufsize(1024*1024), _buf(_bufsize), + _lastReadPos(0), + _nextReadPos(0), _bufused(0), _bufpos(0) { @@ -48,14 +50,14 @@ bool FileReader::Reset() { _file->clear(); - _file->seekg(0); - return bool(*_file); + return SetFilePos(0); } bool FileReader::SetFilePos(int64_t pos) { - _bufpos = 0; + _bufused = _bufpos = 0; + _lastReadPos = _nextReadPos = pos; _file->seekg(pos); return bool(*_file); } @@ -68,21 +70,22 @@ FileReader::GetFileSize() } uint64_t -FileReader::FindNewline(int64_t pos) +FileReader::FindNextLine(int64_t pos) { char buf[100]; SetFilePos(pos); - ssize_t len = ReadLine(buf, 100); - ++_bufpos; - - return pos+len; + ReadLine(buf, 100); + return GetFilePos(); } void FileReader::FillBuffer() { + _lastReadPos = _nextReadPos; _file->read(&_buf[0], _bufsize); - _bufused = _file->gcount(); // may be -1 + auto wasRead = _file->gcount(); // may be -1 + _nextReadPos += wasRead; + _bufused = wasRead; _bufpos = 0; } diff --git a/fbench/src/util/filereader.h b/fbench/src/util/filereader.h index b553c73a262..7fd1943d65d 100644 --- a/fbench/src/util/filereader.h +++ b/fbench/src/util/filereader.h @@ -21,6 +21,8 @@ private: std::istream *_file; int _bufsize; std::vector<char> _buf; + uint64_t _lastReadPos; + uint64_t _nextReadPos; int _bufused; int _bufpos; @@ -57,7 +59,7 @@ public: { if(_bufpos == _bufused) FillBuffer(); - return (_bufused > _bufpos) ? _buf[_bufpos++] & 0x0ff : -1; + return (_bufused > _bufpos) ? (_buf[_bufpos++] & 0x0ff) : -1; } /** @@ -94,14 +96,14 @@ public: int64_t GetFileSize(); /** - * @returns _bufpos + * @return current position in file **/ - uint64_t GetBufPos() const { return _bufpos; } + uint64_t GetFilePos() const { return _lastReadPos + _bufpos; } /** - * @returns offset of next newline from pos + * @return offset to start of next line from pos **/ - uint64_t FindNewline(int64_t pos); + uint64_t FindNextLine(int64_t pos); /** * Read the next line of text from the the currently open file into diff --git a/filedistribution/src/apps/status/status-filedistribution.cpp b/filedistribution/src/apps/status/status-filedistribution.cpp index 697c7b504af..256266efa54 100644 --- a/filedistribution/src/apps/status/status-filedistribution.cpp +++ b/filedistribution/src/apps/status/status-filedistribution.cpp @@ -1,18 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("status-filedistribution"); - -#include <iostream> -#include <map> -#include <thread> - -#include <boost/program_options.hpp> #include <vespa/filedistribution/model/zkfacade.h> #include <vespa/filedistribution/model/filedistributionmodel.h> #include <vespa/filedistribution/model/filedistributionmodelimpl.h> #include <zookeeper/zookeeper.h> +#include <boost/program_options.hpp> + +#include <vespa/log/log.h> +#include <iostream> +#include <map> +#include <thread> + +LOG_SETUP("status-filedistribution"); using namespace filedistribution; using namespace std::literals; diff --git a/filedistribution/src/tests/common/testCommon.cpp b/filedistribution/src/tests/common/testCommon.cpp index 1902d4ec7db..704c9647a50 100644 --- a/filedistribution/src/tests/common/testCommon.cpp +++ b/filedistribution/src/tests/common/testCommon.cpp @@ -2,7 +2,6 @@ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN -#include <vespa/fastos/fastos.h> #include <vespa/filedistribution/common/buffer.h> #include <boost/test/unit_test.hpp> diff --git a/filedistribution/src/tests/filedbmodelimpl/test-filedistributionmodelimpl.cpp b/filedistribution/src/tests/filedbmodelimpl/test-filedistributionmodelimpl.cpp index 31dd267366c..1ef34788958 100644 --- a/filedistribution/src/tests/filedbmodelimpl/test-filedistributionmodelimpl.cpp +++ b/filedistribution/src/tests/filedbmodelimpl/test-filedistributionmodelimpl.cpp @@ -2,7 +2,6 @@ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #define BOOST_TEST_MODULE filedbmodelimpl test -#include <vespa/fastos/fastos.h> #include <boost/test/unit_test.hpp> #include <iostream> diff --git a/filedistribution/src/tests/rpc/testfileprovider.cpp b/filedistribution/src/tests/rpc/testfileprovider.cpp index 6be172d0afd..acf12086437 100644 --- a/filedistribution/src/tests/rpc/testfileprovider.cpp +++ b/filedistribution/src/tests/rpc/testfileprovider.cpp @@ -2,11 +2,11 @@ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN -#include <vespa/fastos/fastos.h> #include "mockfileprovider.h" #include <vespa/filedistribution/rpc/filedistributorrpc.h> #include <vespa/frtstream/frtclientstream.h> -#include <iostream> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/target.h> #include <boost/test/unit_test.hpp> namespace fd = filedistribution; diff --git a/filedistribution/src/tests/status/test-status.cpp b/filedistribution/src/tests/status/test-status.cpp index 4fbda2cb9c3..a3de52dc63b 100644 --- a/filedistribution/src/tests/status/test-status.cpp +++ b/filedistribution/src/tests/status/test-status.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN -#include <vespa/fastos/fastos.h> #include <boost/test/unit_test.hpp> #include <vespa/filedistribution/model/zkfacade.h> diff --git a/filedistribution/src/tests/zkfiledbmodel/test-zkfiledbmodel.cpp b/filedistribution/src/tests/zkfiledbmodel/test-zkfiledbmodel.cpp index 690cb73da11..77c5f5b9034 100644 --- a/filedistribution/src/tests/zkfiledbmodel/test-zkfiledbmodel.cpp +++ b/filedistribution/src/tests/zkfiledbmodel/test-zkfiledbmodel.cpp @@ -2,7 +2,6 @@ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #define BOOST_TEST_MODULE zkfiledbmodel test -#include <vespa/fastos/fastos.h> #include <boost/test/unit_test.hpp> #include <iostream> diff --git a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp index 3c6a265941a..ddef7a2d175 100644 --- a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp +++ b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "componentsdeleter.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".componentsdeleter"); diff --git a/filedistribution/src/vespa/filedistribution/common/exception.cpp b/filedistribution/src/vespa/filedistribution/common/exception.cpp index c9bb5f43ab0..93d07d69118 100644 --- a/filedistribution/src/vespa/filedistribution/common/exception.cpp +++ b/filedistribution/src/vespa/filedistribution/common/exception.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "exception.h" namespace filedistribution { diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp index e042ec28870..14cb8844d8b 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/filedistribution/model/zkfacade.h> #include "filedownloader.h" #include "hostname.h" - -#include <iterator> -#include <algorithm> +#include <vespa/filedistribution/model/zkfacade.h> +#include <vespa/vespalib/util/stringfmt.h> #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> @@ -17,6 +14,9 @@ #include <libtorrent/torrent_handle.hpp> #include <libtorrent/bencode.hpp> +#include <iterator> +#include <algorithm> + #include <vespa/log/log.h> LOG_SETUP(".filedownloader"); diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp index 5b6ad4f83ff..2b143b9c083 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".filedownloadermanager"); - #include "filedownloadermanager.h" #include <iterator> #include <sstream> #include <thread> +#include <vespa/log/log.h> +LOG_SETUP(".filedownloadermanager"); + using namespace std::literals; using filedistribution::FileDownloaderManager; diff --git a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp index 7b33632df81..f307ae1fca3 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "hostname.h" +#include "hostname.h" +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/log/log.h> LOG_SETUP(".hostname"); diff --git a/filedistribution/src/vespa/filedistribution/distributor/scheduler.cpp b/filedistribution/src/vespa/filedistribution/distributor/scheduler.cpp index 4f75afb4850..ac05a24ad36 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/scheduler.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/scheduler.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "scheduler.h" -#include <iostream> namespace asio = boost::asio; diff --git a/filedistribution/src/vespa/filedistribution/distributor/state_server_impl.cpp b/filedistribution/src/vespa/filedistribution/distributor/state_server_impl.cpp index 95a0a1a64ca..6ec9c677687 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/state_server_impl.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/state_server_impl.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "state_server_impl.h" namespace filedistribution { diff --git a/filedistribution/src/vespa/filedistribution/manager/createtorrent.cpp b/filedistribution/src/vespa/filedistribution/manager/createtorrent.cpp index 52b05cd78be..551e6e0c1cb 100644 --- a/filedistribution/src/vespa/filedistribution/manager/createtorrent.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/createtorrent.cpp @@ -1,7 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "createtorrent.h" +#include <libtorrent/torrent_info.hpp> +#include <boost/filesystem/convenience.hpp> + #include <iostream> #include <fstream> #include <cmath> @@ -9,9 +11,6 @@ #include <sstream> #include <string> -#include <boost/filesystem/convenience.hpp> -#include "libtorrent/torrent_info.hpp" - namespace fs = boost::filesystem; namespace { diff --git a/filedistribution/src/vespa/filedistribution/manager/filedb.cpp b/filedistribution/src/vespa/filedistribution/manager/filedb.cpp index a5c440b93a0..fc8c8a95442 100644 --- a/filedistribution/src/vespa/filedistribution/manager/filedb.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/filedb.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "filedb.h" #include <boost/filesystem.hpp> diff --git a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp index 6af340a0ff6..0da6588958f 100644 --- a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp @@ -1,16 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "jnistring.h" +#include "field.h" +#include "createtorrent.h" +#include "filedb.h" #include <vespa/filedistribution/manager/com_yahoo_vespa_filedistribution_FileDistributionManager.h> -#include <memory> #include <vespa/filedistribution/model/filedistributionmodel.h> #include <vespa/filedistribution/model/zkfiledbmodel.h> #include <vespa/filedistribution/model/mockfiledistributionmodel.h> #include <vespa/filedistribution/model/zkfacade.h> -#include "jnistring.h" -#include "field.h" -#include "createtorrent.h" -#include "filedb.h" +#include <memory> using namespace filedistribution; diff --git a/filedistribution/src/vespa/filedistribution/manager/jnistring.h b/filedistribution/src/vespa/filedistribution/manager/jnistring.h index aa5d459232e..295cfc0fe8b 100644 --- a/filedistribution/src/vespa/filedistribution/manager/jnistring.h +++ b/filedistribution/src/vespa/filedistribution/manager/jnistring.h @@ -3,7 +3,7 @@ #include <exception> #include <string> - +#include <vector> #include <jni.h> namespace filedistribution { diff --git a/filedistribution/src/vespa/filedistribution/manager/stderr_logfwd.cpp b/filedistribution/src/vespa/filedistribution/manager/stderr_logfwd.cpp index a5041554114..0907509bbab 100644 --- a/filedistribution/src/vespa/filedistribution/manager/stderr_logfwd.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/stderr_logfwd.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/filedistribution/common/logfwd.h> -#include <stdarg.h> #include <iostream> -#include <stdio.h> - +#include <vector> void filedistribution::logfwd::log_forward(LogLevel level, const char* file, int line, const char* fmt, ...) diff --git a/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp b/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp index 7829744949f..05ee8db9060 100644 --- a/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp +++ b/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "deployedfilestodownload.h" +#include "deployedfilestodownload.h" +#include <vespa/filedistribution/common/logfwd.h> #include <sstream> #include <iterator> -#include <vespa/filedistribution/common/logfwd.h> - using filedistribution::DeployedFilesToDownload; using filedistribution::Path; diff --git a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp index 3b4d043a30b..15ff198b8d5 100644 --- a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp +++ b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp @@ -4,6 +4,7 @@ #include "zkfiledbmodel.h" #include "deployedfilestodownload.h" #include "filedistributionmodelimpl.h" +#include <vespa/vespalib/util/stringfmt.h> #include <boost/filesystem.hpp> #include <zookeeper/zookeeper.h> diff --git a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp index 2400eae9eb7..28e0e82f79a 100644 --- a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp +++ b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp @@ -2,15 +2,15 @@ #include "zkfacade.h" #include <vespa/vespalib/net/socket_address.h> -#include <sstream> -#include <thread> -#include <boost/function_output_iterator.hpp> - -#include <zookeeper/zookeeper.h> #include <vespa/filedistribution/common/logfwd.h> #include <vespa/defaults.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <zookeeper/zookeeper.h> +#include <sstream> +#include <thread> +#include <boost/function_output_iterator.hpp> typedef std::unique_lock<std::mutex> UniqueLock; diff --git a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp index 5198b6bbeb7..4030b777afa 100644 --- a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp +++ b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp @@ -5,6 +5,7 @@ #include "zkfiledbmodel.h" #include "deployedfilestodownload.h" #include <vespa/filedistribution/common/logfwd.h> +#include <vespa/vespalib/util/stringfmt.h> #include <sys/file.h> #include <ostream> #include <algorithm> diff --git a/fnet/src/examples/frt/rpc/.gitignore b/fnet/src/examples/frt/rpc/.gitignore index 91218a0868b..907b6c8a249 100644 --- a/fnet/src/examples/frt/rpc/.gitignore +++ b/fnet/src/examples/frt/rpc/.gitignore @@ -5,12 +5,12 @@ echo_server rpc_callback_client rpc_callback_server rpc_client -rpc_info -rpc_invoke -rpc_proxy rpc_server fnet_echo_client_app fnet_rpc_callback_client_app fnet_rpc_callback_server_app fnet_rpc_client_app fnet_rpc_server_app +vespa-rpc-info +vespa-rpc-invoke +vespa-rpc-proxy diff --git a/fnet/src/examples/frt/rpc/CMakeLists.txt b/fnet/src/examples/frt/rpc/CMakeLists.txt index aae76bdcd4f..0d71ba4f91e 100644 --- a/fnet/src/examples/frt/rpc/CMakeLists.txt +++ b/fnet/src/examples/frt/rpc/CMakeLists.txt @@ -2,28 +2,25 @@ vespa_add_executable(fnet_rpc_server_app SOURCES rpc_server.cpp - INSTALL bin DEPENDS fnet ) vespa_add_executable(fnet_rpc_client_app SOURCES rpc_client.cpp - INSTALL bin DEPENDS fnet ) vespa_add_executable(fnet_echo_client_app SOURCES echo_client.cpp - INSTALL bin DEPENDS fnet ) vespa_add_executable(fnet_rpc_info_app SOURCES rpc_info.cpp - OUTPUT_NAME rpc_info + OUTPUT_NAME vespa-rpc-info INSTALL bin DEPENDS fnet @@ -31,7 +28,7 @@ vespa_add_executable(fnet_rpc_info_app vespa_add_executable(fnet_rpc_proxy_app SOURCES rpc_proxy.cpp - OUTPUT_NAME rpc_proxy + OUTPUT_NAME vespa-rpc-proxy INSTALL bin DEPENDS fnet @@ -39,21 +36,19 @@ vespa_add_executable(fnet_rpc_proxy_app vespa_add_executable(fnet_rpc_callback_server_app SOURCES rpc_callback_server.cpp - INSTALL bin DEPENDS fnet ) vespa_add_executable(fnet_rpc_callback_client_app SOURCES rpc_callback_client.cpp - INSTALL bin DEPENDS fnet ) vespa_add_executable(fnet_rpc_invoke_app SOURCES rpc_invoke.cpp - OUTPUT_NAME rpc_invoke + OUTPUT_NAME vespa-rpc-invoke INSTALL bin DEPENDS fnet diff --git a/fnet/src/examples/ping/CMakeLists.txt b/fnet/src/examples/ping/CMakeLists.txt index 7354ec9cf48..ff635799848 100644 --- a/fnet/src/examples/ping/CMakeLists.txt +++ b/fnet/src/examples/ping/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(fnet_pingserver_app SOURCES packets.cpp pingserver.cpp - INSTALL bin DEPENDS fnet ) @@ -11,7 +10,6 @@ vespa_add_executable(fnet_pingclient_app SOURCES packets.cpp pingclient.cpp - INSTALL bin DEPENDS fnet ) diff --git a/fnet/src/examples/proxy/CMakeLists.txt b/fnet/src/examples/proxy/CMakeLists.txt index bc5dce755a7..45710fc0779 100644 --- a/fnet/src/examples/proxy/CMakeLists.txt +++ b/fnet/src/examples/proxy/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_executable(fnet_proxy_app SOURCES proxy.cpp - INSTALL bin DEPENDS fnet ) diff --git a/fnet/src/examples/timeout/CMakeLists.txt b/fnet/src/examples/timeout/CMakeLists.txt index 5b9514ab0c8..2f4a710c1f2 100644 --- a/fnet/src/examples/timeout/CMakeLists.txt +++ b/fnet/src/examples/timeout/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_executable(fnet_timeout_app SOURCES timeout.cpp - INSTALL bin DEPENDS fnet ) diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp index ca77ab6f73b..8b0edb4476f 100644 --- a/fnet/src/tests/sync_execute/sync_execute.cpp +++ b/fnet/src/tests/sync_execute/sync_execute.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/iexecutable.h> +#include <vespa/fastos/thread.h> struct DoIt : public FNET_IExecutable { vespalib::Gate gate; diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h index 2441fb081d3..a834606d92e 100644 --- a/fnet/src/vespa/fnet/frt/rpcrequest.h +++ b/fnet/src/vespa/fnet/frt/rpcrequest.h @@ -126,8 +126,8 @@ public: void SetMethodName(const char *methodName, uint32_t len); void SetMethodName(const char *methodName); - uint32_t GetMethodNameLen() { return _methodNameLen; } - const char *GetMethodName() { return _methodName; } + uint32_t GetMethodNameLen() const { return _methodNameLen; } + const char *GetMethodName() const { return _methodName; } void Print(uint32_t indent = 0); diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index 71f02754a92..dd7a3246090 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -2,6 +2,7 @@ #include "scheduler.h" #include "task.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".fnet.scheduler"); diff --git a/frtstream/src/example/test.cpp b/frtstream/src/example/test.cpp index 5af115f9978..0c230f2d5bf 100644 --- a/frtstream/src/example/test.cpp +++ b/frtstream/src/example/test.cpp @@ -1,14 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include <vespa/frtstream/frtclientstream.h> + #include <iostream> #include <csignal> #include <vector> #include <string> #include <set> -#include <vespa/frtstream/frtclientstream.h> - - using namespace std; using frtstream::FrtClientStream; using frtstream::Method; diff --git a/frtstream/src/vespa/frtstream/frtclientstream.cpp b/frtstream/src/vespa/frtstream/frtclientstream.cpp index 142b0d1fda0..e9e17b8e02b 100644 --- a/frtstream/src/vespa/frtstream/frtclientstream.cpp +++ b/frtstream/src/vespa/frtstream/frtclientstream.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "frtclientstream.h" +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> using namespace fnet; diff --git a/frtstream/src/vespa/frtstream/frtclientstream.h b/frtstream/src/vespa/frtstream/frtclientstream.h index b47f1399d0f..b06a6af890d 100644 --- a/frtstream/src/vespa/frtstream/frtclientstream.h +++ b/frtstream/src/vespa/frtstream/frtclientstream.h @@ -2,16 +2,11 @@ #pragma once -//Requires gcc 3 or higher -#if ! defined(__GNUC__) || (__GNUC__ > 2) - - -#include <vespa/frtstream/frtstream.h> - +#include "frtstream.h" +#include <vespa/fnet/frt/supervisor.h> namespace frtstream { - class FrtClientStream : public FrtStream { FRT_Supervisor supervisor; FRT_RPCRequest* request; @@ -34,7 +29,3 @@ public: } //end namespace frtstream -#else -#error "Requires gcc 3 or higher" -#endif - diff --git a/frtstream/src/vespa/frtstream/frtstream.h b/frtstream/src/vespa/frtstream/frtstream.h index cd50b1eae32..43bcf1aa8f1 100644 --- a/frtstream/src/vespa/frtstream/frtstream.h +++ b/frtstream/src/vespa/frtstream/frtstream.h @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/fnet/frt/values.h> #include <string> #include <memory> #include <algorithm> #include <iosfwd> #include <iterator> -#include <vespa/fastos/types.h> -#include <vespa/fnet/frt/frt.h> - - namespace frtstream { + class ConnectionException{}; class InvokationException{ public: @@ -85,8 +83,6 @@ public: }; - - } //end namespace frtstream diff --git a/frtstream/src/vespa/frtstream/frtstreamTemplateImp.hpp b/frtstream/src/vespa/frtstream/frtstreamTemplateImp.hpp index 4577f8a7e78..0f89b42e401 100644 --- a/frtstream/src/vespa/frtstream/frtstreamTemplateImp.hpp +++ b/frtstream/src/vespa/frtstream/frtstreamTemplateImp.hpp @@ -5,7 +5,6 @@ //Can be moved when(if ever) the export functionality //is implemented in the target compiler. - namespace frtstream { template<class U> diff --git a/fsa/src/apps/fsadump/.gitignore b/fsa/src/apps/fsadump/.gitignore index 36c86d6022c..9c0cf4cb39b 100644 --- a/fsa/src/apps/fsadump/.gitignore +++ b/fsa/src/apps/fsadump/.gitignore @@ -1 +1 @@ -fsadump +vespa-fsadump diff --git a/fsa/src/apps/fsadump/CMakeLists.txt b/fsa/src/apps/fsadump/CMakeLists.txt index 069bdfb379b..cd9113be3dd 100644 --- a/fsa/src/apps/fsadump/CMakeLists.txt +++ b/fsa/src/apps/fsadump/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(fsa_fsadump_app SOURCES fsadump.cpp - OUTPUT_NAME fsadump + OUTPUT_NAME vespa-fsadump INSTALL bin DEPENDS fsa diff --git a/fsa/src/apps/fsainfo/.gitignore b/fsa/src/apps/fsainfo/.gitignore index fc50ebfe566..c515ae6d54c 100644 --- a/fsa/src/apps/fsainfo/.gitignore +++ b/fsa/src/apps/fsainfo/.gitignore @@ -1 +1 @@ -fsainfo +vespa-fsainfo diff --git a/fsa/src/apps/fsainfo/CMakeLists.txt b/fsa/src/apps/fsainfo/CMakeLists.txt index c16332ed20b..30d9c173d7f 100644 --- a/fsa/src/apps/fsainfo/CMakeLists.txt +++ b/fsa/src/apps/fsainfo/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(fsa_fsainfo_app SOURCES fsainfo.cpp - OUTPUT_NAME fsainfo + OUTPUT_NAME vespa-fsainfo INSTALL bin DEPENDS fsa diff --git a/fsa/src/apps/makefsa/.gitignore b/fsa/src/apps/makefsa/.gitignore index 1ea7393bec3..2ae23b715f0 100644 --- a/fsa/src/apps/makefsa/.gitignore +++ b/fsa/src/apps/makefsa/.gitignore @@ -1 +1 @@ -makefsa +vespa-makefsa diff --git a/fsa/src/apps/makefsa/CMakeLists.txt b/fsa/src/apps/makefsa/CMakeLists.txt index 80002338479..60231e01591 100644 --- a/fsa/src/apps/makefsa/CMakeLists.txt +++ b/fsa/src/apps/makefsa/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(fsa_makefsa_app SOURCES makefsa.cpp - OUTPUT_NAME makefsa + OUTPUT_NAME vespa-makefsa INSTALL bin DEPENDS fsa diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java index 1617b715c68..74e06bc7050 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class TokenizeExpression extends Expression { diff --git a/install_java.cmake b/install_java.cmake index 361a07dad5d..408582b387e 100644 --- a/install_java.cmake +++ b/install_java.cmake @@ -46,6 +46,7 @@ install_fat_java_artifact(simplemetrics) install_fat_java_artifact(standalone-container) install_fat_java_artifact(vespa-http-client) install_fat_java_artifact(vespaclient-container-plugin) +install_fat_java_artifact(vespaclient-java) install_fat_java_artifact(zkfacade) vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin) @@ -57,6 +58,15 @@ vespa_install_script(config-model/src/main/perl/expand-config.pl bin) vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin) vespa_install_script(config-model/src/main/sh/validate-application bin) vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) +vespa_install_script(vespaclient-java/src/main/sh/vds-document-statistics.sh vds-document-statistics bin) +vespa_install_script(vespaclient-java/src/main/sh/vdsstat.sh vdsstat bin) +vespa_install_script(vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin) +vespa_install_script(vespaclient-java/src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin) +vespa_install_script(vespaclient-java/src/main/sh/vespadestination.sh vespadestination bin) +vespa_install_script(vespaclient-java/src/main/sh/vespafeeder.sh vespafeeder bin) +vespa_install_script(vespaclient-java/src/main/sh/vespaget.sh vespaget bin) +vespa_install_script(vespaclient-java/src/main/sh/vespavisit.sh vespavisit bin) +vespa_install_script(vespaclient-java/src/main/sh/vespavisittarget.sh vespavisittarget bin) vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin) diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 6227600f24c..bc6240919e9 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -120,6 +120,12 @@ <scope>compile</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <!-- This seems odd. Used for export-package parsing. Lazy stuff. Should be separated out. --> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java index 3ed10170420..82029e85cd6 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/AbstractResource.java @@ -69,6 +69,7 @@ public abstract class AbstractResource implements SharedResource { return new DebugResourceReference(this, referenceStack); } + @Override public void release() { initialCreationReference.close(); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java index a296bd1e327..692c669780f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java @@ -9,7 +9,6 @@ import com.yahoo.jdisc.application.BindingSet; import com.yahoo.jdisc.application.BindingSetSelector; import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.application.ResourcePool; -import com.yahoo.jdisc.application.UriPattern; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.BindingSetNotFoundException; import com.yahoo.jdisc.service.CurrentContainer; @@ -18,12 +17,15 @@ import com.yahoo.jdisc.service.ServerProvider; import java.net.URI; import java.util.Map; +import java.util.logging.Logger; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> */ public class ActiveContainer extends AbstractResource implements CurrentContainer { + private static final Logger log = Logger.getLogger(ActiveContainer.class.getName()); + private final ContainerTermination termination; private final Injector guiceInjector; private final Iterable<ServerProvider> serverProviders; @@ -35,21 +37,15 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine public ActiveContainer(ContainerBuilder builder) { serverProviders = builder.serverProviders().activate(); - for (SharedResource resource : serverProviders) { - resourceReferences.retain(resource); - } + serverProviders.forEach(resourceReferences::retain); serverBindings = builder.activateServerBindings(); - for (BindingSet<RequestHandler> set : serverBindings.values()) { - for (Map.Entry<UriPattern, RequestHandler> entry : set) { - resourceReferences.retain(entry.getValue()); - } - } + serverBindings.forEach( + (ignoredName, bindingSet) -> bindingSet.forEach( + binding -> resourceReferences.retain(binding.getValue()))); clientBindings = builder.activateClientBindings(); - for (BindingSet<RequestHandler> set : clientBindings.values()) { - for (Map.Entry<UriPattern, RequestHandler> entry : set) { - resourceReferences.retain(entry.getValue()); - } - } + clientBindings.forEach( + (ignoredName, bindingSet) -> bindingSet.forEach( + binding -> resourceReferences.retain(binding.getValue()))); bindingSetSelector = builder.getInstance(BindingSetSelector.class); timeoutMgr = builder.getInstance(TimeoutManagerImpl.class); timeoutMgr.start(); @@ -74,7 +70,11 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine @Override protected void finalize() throws Throwable { try { - if (retainCount() > 0) { + int retainCount = retainCount(); + if (retainCount > 0) { + log.warning(this + ".destroy() invoked from finalize() not through ApplicationLoader. " + + "This is an indication of either a resource leak or invalid use of reference counting. " + + "Retained references as this moment: " + retainCount); destroy(); } } finally { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java index c6d6efd0ee9..c6acde814eb 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.application.ContainerThread; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.jdisc.statistics.ActiveContainerStatistics; import java.util.concurrent.ThreadFactory; @@ -28,6 +29,7 @@ class ApplicationEnvironmentModule extends AbstractModule { bind(CurrentContainer.class).toInstance(loader); bind(OsgiFramework.class).toInstance(loader.osgiFramework()); bind(ThreadFactory.class).to(ContainerThread.Factory.class); + bind(ActiveContainerStatistics.class).toInstance(loader.getActiveContainerStatistics()); } @Provides diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 2dd7f7eb879..4e63bc77c9a 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -5,9 +5,17 @@ import com.google.inject.AbstractModule; import com.google.inject.Injector; import com.google.inject.Module; import com.yahoo.jdisc.AbstractResource; -import com.yahoo.jdisc.application.*; +import com.yahoo.jdisc.application.Application; +import com.yahoo.jdisc.application.ApplicationNotReadyException; +import com.yahoo.jdisc.application.ContainerActivator; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.application.DeactivatedContainer; +import com.yahoo.jdisc.application.GuiceRepository; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.application.OsgiHeader; import com.yahoo.jdisc.service.ContainerNotReadyException; import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.jdisc.statistics.ActiveContainerStatistics; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleException; @@ -28,11 +36,13 @@ import java.util.logging.Logger; public class ApplicationLoader implements BootstrapLoader, ContainerActivator, CurrentContainer { private static final Logger log = Logger.getLogger(ApplicationLoader.class.getName()); + private final OsgiFramework osgiFramework; private final GuiceRepository guiceModules = new GuiceRepository(); private final AtomicReference<ActiveContainer> containerRef = new AtomicReference<>(); private final Object appLock = new Object(); private final List<Bundle> appBundles = new ArrayList<>(); + private final ActiveContainerStatistics statistics = new ActiveContainerStatistics(); private Application application; private ApplicationInUseTracker applicationInUseTracker; @@ -62,9 +72,11 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C } prev = containerRef.getAndSet(next); + statistics.onActivated(next); if (prev == null) { return null; } + statistics.onDeactivated(prev); } prev.release(); DeactivatedContainer deactivatedContainer = prev.shutdown(); @@ -82,11 +94,9 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C Thread.sleep(TimeUnit.MILLISECONDS.convert(currentWaitTimeSeconds, TimeUnit.SECONDS)) ); + statistics.printSummaryToLog(); final ActiveContainer prevContainer = prevContainerReference.get(); - if (prevContainer == null) { - return; - } - if (prevContainer.retainCount() == 0) { + if (prevContainer == null || prevContainer.retainCount() == 0) { return; } log.warning("Previous container not terminated in the last " + totalTimeWaited + " seconds." @@ -231,6 +241,10 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C } } + public ActiveContainerStatistics getActiveContainerStatistics() { + return statistics; + } + public OsgiFramework osgiFramework() { return osgiFramework; } @@ -258,4 +272,5 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C } } } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java index 03110782d23..33d03d29f7d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BootstrapDaemon.java @@ -1,10 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; +import com.yahoo.protect.Process; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,6 +40,56 @@ public class BootstrapDaemon implements Daemon { return loader; } + private static class WatchDog implements Runnable { + final String name; + final CountDownLatch complete; + final long timeout; + final TimeUnit timeUnit; + WatchDog(String name, CountDownLatch complete, long timeout, TimeUnit timeUnit) { + this.name = name; + this.complete = complete; + this.timeout = timeout; + this.timeUnit = timeUnit; + } + @Override + public void run() { + boolean dumpStack; + try { + dumpStack = !complete.await(timeout, timeUnit); + } catch (InterruptedException e) { + return; + } + if (dumpStack) { + log.warning("The watchdog for BootstrapDaemon." + name + " detected that it had not completed in " + + timeUnit.toMillis(timeout) + "ms. Dumping stack."); + Process.dumpThreads(); + } + } + } + private interface MyRunnable { + void run() throws Exception; + } + private void startWithWatchDog(String name, long timeout, TimeUnit timeUnit, MyRunnable task) throws Exception { + CountDownLatch complete = new CountDownLatch(1); + Thread thread = new Thread(new WatchDog(name, complete, timeout, timeUnit), name); + thread.setDaemon(true); + thread.start(); + try { + task.run(); + } catch (Exception e) { + log.log(Level.WARNING, "Exception caught during BootstrapDaemon." + name, e); + throw e; + } catch (Error e) { + log.log(Level.WARNING, "Error caught during BootstrapDaemon." + name, e); + throw e; + } catch (Throwable thrown) { + log.log(Level.WARNING, "Throwable caught during BootstrapDaemon." + name, thrown); + } finally { + complete.countDown(); + thread.join(); + } + } + @Override public void init(DaemonContext context) throws Exception { String[] args = context.getArguments(); @@ -46,7 +99,7 @@ public class BootstrapDaemon implements Daemon { bundleLocation = args[0]; if (privileged) { log.finer("Initializing application with privileges."); - loader.init(bundleLocation, true); + startWithWatchDog("init", 60, TimeUnit.SECONDS, () -> loader.init(bundleLocation, true)); } } @@ -55,9 +108,9 @@ public class BootstrapDaemon implements Daemon { try { if (!privileged) { log.finer("Initializing application without privileges."); - loader.init(bundleLocation, false); + startWithWatchDog("init", 60, TimeUnit.SECONDS, () -> loader.init(bundleLocation, false)); } - loader.start(); + startWithWatchDog("start", 60, TimeUnit.SECONDS, () -> loader.start()); } catch (Exception e) { try { log.log(Level.SEVERE, "Failed starting container", e); @@ -70,12 +123,15 @@ public class BootstrapDaemon implements Daemon { @Override public void stop() throws Exception { - loader.stop(); + startWithWatchDog("stop", 60, TimeUnit.SECONDS, () -> loader.stop()); } @Override public void destroy() { - loader.destroy(); + try { + startWithWatchDog("destroy", 60, TimeUnit.SECONDS, () -> loader.destroy()); + } catch (Exception e) { + } } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index afe43718bc5..3dbc4af5422 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.yahoo.container.plugin.bundle.AnalyzeBundle; import com.yahoo.container.plugin.bundle.TransformExportPackages; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import org.apache.felix.framework.util.Util; @@ -38,6 +37,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.application,") .append("com.yahoo.jdisc.handler,") .append("com.yahoo.jdisc.service,") + .append("com.yahoo.jdisc.statistics,") .append("javax.inject;version=1.0.0,") // Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept,") .append("org.aopalliance.aop,") diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java new file mode 100644 index 00000000000..2fac3b4024b --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ActiveContainerStatistics.java @@ -0,0 +1,132 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.statistics; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.core.ActiveContainer; + +import java.time.Instant; +import java.util.List; +import java.util.WeakHashMap; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; + +/** + * Tracks statistics on stale {@link ActiveContainer} instances. + * + * @author bjorncs + */ +public class ActiveContainerStatistics { + public interface Metrics { + String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total"; + String DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES = "jdisc.deactivated_containers.with_retained_refs"; + } + + private static final Logger log = Logger.getLogger(ActiveContainerStatistics.class.getName()); + + private final WeakHashMap<ActiveContainer, ActiveContainerStats> activeContainers = new WeakHashMap<>(); + private final Object monitor = new Object(); + + public void emitMetrics(Metric metric) { + synchronized (monitor) { + DeactivatedContainerMetrics metrics = deactivatedContainerStream() + .collect( + DeactivatedContainerMetrics::new, + DeactivatedContainerMetrics::aggregate, + DeactivatedContainerMetrics::merge); + + metric.set(Metrics.TOTAL_DEACTIVATED_CONTAINERS, metrics.deactivatedContainerCount, null); + metric.set(Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES, metrics.deactivatedContainersWithRetainedRefsCount, null); + } + } + + public void onActivated(ActiveContainer activeContainer) { + synchronized (monitor) { + activeContainers.put(activeContainer, new ActiveContainerStats(Instant.now())); + } + } + + public void onDeactivated(ActiveContainer activeContainer) { + synchronized (monitor) { + ActiveContainerStats containerStats = activeContainers.get(activeContainer); + if (containerStats == null) { + throw new IllegalStateException("onActivated() has not been called for container: " + activeContainer); + } + containerStats.timeDeactivated = Instant.now(); + } + } + + public void printSummaryToLog() { + synchronized (monitor) { + List<DeactivatedContainer> deactivatedContainers = deactivatedContainerStream().collect(toList()); + if (deactivatedContainers.isEmpty()) return; + + log.warning( + "Multiple instances of ActiveContainer leaked! " + deactivatedContainers.size() + + " instances are still present."); + deactivatedContainers.stream() + .map(c -> " - " + c.toSummaryString()) + .forEach(log::warning); + } + } + + private Stream<DeactivatedContainer> deactivatedContainerStream() { + synchronized (monitor) { + return activeContainers.entrySet().stream() + .filter(e -> e.getValue().isDeactivated()) + .map(e -> new DeactivatedContainer(e.getKey(), e.getValue().timeActivated, e.getValue().timeDeactivated)); + } + } + + private static class ActiveContainerStats { + public final Instant timeActivated; + public Instant timeDeactivated; + + public ActiveContainerStats(Instant timeActivated) { + this.timeActivated = timeActivated; + } + + public boolean isDeactivated() { + return timeDeactivated != null; + } + } + + private static class DeactivatedContainer { + public final ActiveContainer activeContainer; + public final Instant timeActivated; + public final Instant timeDeactivated; + + public DeactivatedContainer(ActiveContainer activeContainer, Instant timeActivated, Instant timeDeactivated) { + this.activeContainer = activeContainer; + this.timeActivated = timeActivated; + this.timeDeactivated = timeDeactivated; + } + + public String toSummaryString() { + return String.format("%s: timeActivated=%s, timeDeactivated=%s, retainCount=%d", + activeContainer.toString(), + timeActivated.toString(), + timeDeactivated.toString(), + activeContainer.retainCount()); + } + } + + private static class DeactivatedContainerMetrics { + public int deactivatedContainerCount = 0; + public int deactivatedContainersWithRetainedRefsCount = 0; + + public void aggregate(DeactivatedContainer deactivatedContainer) { + ++deactivatedContainerCount; + if (deactivatedContainer.activeContainer.retainCount() > 0) { + ++deactivatedContainersWithRetainedRefsCount; + } + } + + public DeactivatedContainerMetrics merge(DeactivatedContainerMetrics other) { + deactivatedContainerCount += other.deactivatedContainerCount; + deactivatedContainersWithRetainedRefsCount += other.deactivatedContainersWithRetainedRefsCount; + return this; + } + } +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java new file mode 100644 index 00000000000..4289bcbaa9d --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java @@ -0,0 +1,4 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@com.yahoo.osgi.annotation.ExportPackage +package com.yahoo.jdisc.statistics; diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java new file mode 100644 index 00000000000..0a3482b705a --- /dev/null +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/statistics/ActiveContainerStatisticsTest.java @@ -0,0 +1,79 @@ +package com.yahoo.jdisc.statistics; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.core.ActiveContainer; +import com.yahoo.jdisc.test.TestDriver; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author bjorncs + */ +public class ActiveContainerStatisticsTest { + + @Test + public void counts_deactivated_activecontainers() { + TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); + ActiveContainerStatistics stats = new ActiveContainerStatistics(); + MockMetric metric = new MockMetric(); + + ActiveContainer containerWithoutRetainedResources = new ActiveContainer(driver.newContainerBuilder()); + + stats.onActivated(containerWithoutRetainedResources); + stats.emitMetrics(metric); + assertEquals(0, metric.totalCount); + assertEquals(0, metric.withRetainedReferencesCount); + + stats.onDeactivated(containerWithoutRetainedResources); + containerWithoutRetainedResources.release(); + stats.emitMetrics(metric); + assertEquals(1, metric.totalCount); + assertEquals(0, metric.withRetainedReferencesCount); + + ActiveContainer containerWithRetainedResources = new ActiveContainer(driver.newContainerBuilder()); + + try (ResourceReference ignoredRef = containerWithRetainedResources.refer()) { + stats.onActivated(containerWithRetainedResources); + stats.onDeactivated(containerWithRetainedResources); + containerWithRetainedResources.release(); + stats.emitMetrics(metric); + assertEquals(2, metric.totalCount); + assertEquals(1, metric.withRetainedReferencesCount); + } + + } + + private static class MockMetric implements Metric { + public int totalCount; + public int withRetainedReferencesCount; + + @Override + public void set(String key, Number val, Context ctx) { + switch (key) { + case ActiveContainerStatistics.Metrics.TOTAL_DEACTIVATED_CONTAINERS: + totalCount = val.intValue(); + break; + case ActiveContainerStatistics.Metrics.DEACTIVATED_CONTAINERS_WITH_RETAINED_REFERENCES: + withRetainedReferencesCount = val.intValue(); + break; + default: + throw new UnsupportedOperationException(); + } + } + + @Override + public void add(String key, Number val, Context ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + throw new UnsupportedOperationException(); + } + } + +} diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 6684c219227..3e78fd62d9f 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -172,6 +172,7 @@ jetty-continuation-${jetty.version}.jar, jetty-http-${jetty.version}.jar, jetty-io-${jetty.version}.jar, + jetty-jmx-${jetty.version}.jar, jetty-security-${jetty.version}.jar, jetty-server-${jetty.version}.jar, jetty-servlet-${jetty.version}.jar, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index 6c4759915b2..d203f4e618d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -307,7 +307,6 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { return newServerRequest(container, uri, method, version, null); } - @SuppressWarnings("deprecation") public static HttpRequest newServerRequest(CurrentContainer container, URI uri, Method method, Version version, SocketAddress remoteAddress) { return new HttpRequest(container, uri, method, version, remoteAddress, null); @@ -327,7 +326,6 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { return newClientRequest(parent, uri, method, Version.HTTP_1_1); } - @SuppressWarnings("deprecation") public static HttpRequest newClientRequest(Request parent, URI uri, Method method, Version version) { return new HttpRequest(parent, uri, method, version); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index 2af7646e905..e99ea7bc799 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -87,6 +87,7 @@ class JDiscHttpServlet extends HttpServlet { * Override to set connector attribute before the request becomes an upgrade request in the web socket case. * (After the upgrade, the HttpConnection is no longer available.) */ + @SuppressWarnings("deprecation") @Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { request.setAttribute(JDiscServerConnector.REQUEST_ATTRIBUTE, getConnector(request)); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 837f21519f2..a39f0b159c0 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -15,6 +15,8 @@ import com.yahoo.jdisc.http.server.FilterBindings; import com.yahoo.jdisc.service.AbstractServerProvider; import com.yahoo.jdisc.service.CurrentContainer; import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.jmx.ConnectorServer; +import org.eclipse.jetty.jmx.MBeanContainer; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.RequestLog; @@ -37,8 +39,11 @@ import org.osgi.framework.BundleContext; import org.osgi.framework.InvalidSyntaxException; import org.osgi.framework.ServiceReference; +import javax.management.remote.JMXServiceURL; import javax.servlet.DispatcherType; +import java.lang.management.ManagementFactory; import java.net.BindException; +import java.net.MalformedURLException; import java.nio.channels.FileChannel; import java.nio.channels.ServerSocketChannel; import java.nio.file.Path; @@ -62,6 +67,7 @@ import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author bjorncs */ @Beta public class JettyHttpServer extends AbstractServerProvider { @@ -78,15 +84,15 @@ public class JettyHttpServer extends AbstractServerProvider { String NUM_BYTES_RECEIVED = "serverBytesReceived"; String NUM_BYTES_SENT = "serverBytesSent"; - String MANHATTAN_NUM_BYTES_RECEIVED = "http.in.bytes"; - String MANHATTAN_NUM_BYTES_SENT = "http.out.bytes"; + @Deprecated String MANHATTAN_NUM_BYTES_RECEIVED = "http.in.bytes"; + @Deprecated String MANHATTAN_NUM_BYTES_SENT = "http.out.bytes"; String NUM_CONNECTIONS = "serverNumConnections"; /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; String NUM_REQUESTS = "serverNumRequests"; - String MANHATTAN_NUM_REQUESTS = "http.requests"; + @Deprecated String MANHATTAN_NUM_REQUESTS = "http.requests"; String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; String NUM_FAILED_RESPONSES = "serverNumFailedResponses"; @@ -94,10 +100,10 @@ public class JettyHttpServer extends AbstractServerProvider { String NUM_FAILED_WRITES = "serverNumFailedResponseWrites"; String TOTAL_SUCCESSFUL_LATENCY = "serverTotalSuccessfulResponseLatency"; - String MANHATTAN_TOTAL_SUCCESSFUL_LATENCY = "http.latency"; + @Deprecated String MANHATTAN_TOTAL_SUCCESSFUL_LATENCY = "http.latency"; String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; - String MANHATTAN_TIME_TO_FIRST_BYTE = "http.out.firstbytetime"; + @Deprecated String MANHATTAN_TIME_TO_FIRST_BYTE = "http.out.firstbytetime"; String RESPONSES_1XX = "http.status.1xx"; String RESPONSES_2XX = "http.status.2xx"; @@ -106,7 +112,7 @@ public class JettyHttpServer extends AbstractServerProvider { String RESPONSES_5XX = "http.status.5xx"; String STARTED_MILLIS = "serverStartedMillis"; - String MANHATTAN_STARTED_MILLIS = "proc.uptime"; + @Deprecated String MANHATTAN_STARTED_MILLIS = "proc.uptime"; } private final static Logger log = Logger.getLogger(JettyHttpServer.class.getName()); @@ -138,6 +144,7 @@ public class JettyHttpServer extends AbstractServerProvider { initializeJettyLogging(); server = new Server(); + setupJmx(server, serverConfig); ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(serverConfig.maxWorkerThreads()); Map<Path, FileChannel> keyStoreChannels = getKeyStoreFileChannels(osgiFramework.bundleContext()); @@ -193,6 +200,27 @@ public class JettyHttpServer extends AbstractServerProvider { } } + private static void setupJmx(Server server, ServerConfig serverConfig) { + if (serverConfig.jmx().enabled()) { + System.setProperty("java.rmi.server.hostname", "localhost"); + server.addBean( + new MBeanContainer(ManagementFactory.getPlatformMBeanServer())); + server.addBean( + new ConnectorServer( + createJmxLoopbackOnlyServiceUrl(serverConfig.jmx().listenPort()), + "org.eclipse.jetty.jmx:name=rmiconnectorserver")); + } + } + + private static JMXServiceURL createJmxLoopbackOnlyServiceUrl(int port) { + try { + return new JMXServiceURL( + "rmi", "localhost", port, "/jndi/rmi://localhost:" + port + "/jmxrmi"); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + private HandlerCollection getHandlerCollection( ServerConfig serverConfig, ServletPathsConfig servletPathsConfig, @@ -353,6 +381,7 @@ public class JettyHttpServer extends AbstractServerProvider { } + @SuppressWarnings("deprecation") private void setServerMetrics(StatisticsHandler statistics) { long timeSinceStarted = System.currentTimeMillis() - timeStarted; metric.set(Metrics.STARTED_MILLIS, timeSinceStarted, null); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java index e601b31f7ab..583d0548d69 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java @@ -30,6 +30,7 @@ public class MetricReporter { this.requestStartTime = requestStartTime; } + @SuppressWarnings("deprecation") public void successfulWrite(int numBytes) { setTimeToFirstByteFirstTime(); @@ -38,6 +39,7 @@ public class MetricReporter { metric.set(Metrics.MANHATTAN_NUM_BYTES_SENT, numBytes, context); } + @SuppressWarnings("deprecation") private void setTimeToFirstByteFirstTime() { boolean isFirstWrite = firstSetOfTimeToFirstByte.getAndSet(false); if (isFirstWrite) { @@ -51,6 +53,7 @@ public class MetricReporter { metric.add(Metrics.NUM_FAILED_WRITES, 1, context); } + @SuppressWarnings("deprecation") public void successfulResponse() { setTimeToFirstByteFirstTime(); @@ -69,6 +72,7 @@ public class MetricReporter { metric.add(Metrics.NUM_FAILED_RESPONSES, 1, context); } + @SuppressWarnings("deprecation") public void successfulRead(int bytes_received) { metric.set(JettyHttpServer.Metrics.NUM_BYTES_RECEIVED, bytes_received, context); metric.set(JettyHttpServer.Metrics.MANHATTAN_NUM_BYTES_RECEIVED, bytes_received, context); diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def index 01206869862..006a6726a90 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def @@ -26,3 +26,9 @@ filter[].binding string # Max number of threads in pool maxWorkerThreads int default = 200 + +# Enable embedded JMX server. Note: only accessible through the loopback interface. +jmx.enabled bool default = false + +# Listen port for the JMX server. +jmx.listenPort int default = 1099 diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java index bc8f63dcdca..bc4ab071432 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java @@ -580,7 +580,7 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestContentCloseNondeterministicExceptionWithAsyncFailure() throws Throwable { - new TestRunner().expect(anyOf(success(), serverError())) + new TestRunner().expect(anyOf(success(), successNoContent(), serverError())) .execute(); } diff --git a/jdisc_jetty/pom.xml b/jdisc_jetty/pom.xml index 5887a6b7894..ec6ccb0fafb 100644 --- a/jdisc_jetty/pom.xml +++ b/jdisc_jetty/pom.xml @@ -34,6 +34,10 @@ <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-servlets</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + </dependency> </dependencies> <build> <plugins> diff --git a/jrt/src/com/yahoo/jrt/DataArray.java b/jrt/src/com/yahoo/jrt/DataArray.java index 8871b5df6da..aac38d0ff64 100644 --- a/jrt/src/com/yahoo/jrt/DataArray.java +++ b/jrt/src/com/yahoo/jrt/DataArray.java @@ -57,7 +57,8 @@ public class DataArray extends Value public byte[][] asDataArray() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/DataValue.java b/jrt/src/com/yahoo/jrt/DataValue.java index 284bf2e6a2d..15c45ed62e7 100644 --- a/jrt/src/com/yahoo/jrt/DataValue.java +++ b/jrt/src/com/yahoo/jrt/DataValue.java @@ -44,7 +44,8 @@ public class DataValue extends Value public byte[] asData() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/DoubleArray.java b/jrt/src/com/yahoo/jrt/DoubleArray.java index 3758abe9873..4dd6cf2ba4b 100644 --- a/jrt/src/com/yahoo/jrt/DoubleArray.java +++ b/jrt/src/com/yahoo/jrt/DoubleArray.java @@ -47,7 +47,8 @@ public class DoubleArray extends Value public double[] asDoubleArray() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/DoubleValue.java b/jrt/src/com/yahoo/jrt/DoubleValue.java index e7739e80150..8bd3caeb970 100644 --- a/jrt/src/com/yahoo/jrt/DoubleValue.java +++ b/jrt/src/com/yahoo/jrt/DoubleValue.java @@ -41,7 +41,8 @@ public class DoubleValue extends Value public double asDouble() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/FloatArray.java b/jrt/src/com/yahoo/jrt/FloatArray.java index 05556e94276..1dd8cf9832b 100644 --- a/jrt/src/com/yahoo/jrt/FloatArray.java +++ b/jrt/src/com/yahoo/jrt/FloatArray.java @@ -47,7 +47,8 @@ public class FloatArray extends Value public float[] asFloatArray() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/FloatValue.java b/jrt/src/com/yahoo/jrt/FloatValue.java index bfba69f0965..65df64660aa 100644 --- a/jrt/src/com/yahoo/jrt/FloatValue.java +++ b/jrt/src/com/yahoo/jrt/FloatValue.java @@ -42,7 +42,8 @@ public class FloatValue extends Value public float asFloat() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/Int16Array.java b/jrt/src/com/yahoo/jrt/Int16Array.java index fbb7f304872..895d85a5d59 100644 --- a/jrt/src/com/yahoo/jrt/Int16Array.java +++ b/jrt/src/com/yahoo/jrt/Int16Array.java @@ -47,7 +47,8 @@ public class Int16Array extends Value public short[] asInt16Array() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/Int16Value.java b/jrt/src/com/yahoo/jrt/Int16Value.java index 45c10e09f76..704d566d290 100644 --- a/jrt/src/com/yahoo/jrt/Int16Value.java +++ b/jrt/src/com/yahoo/jrt/Int16Value.java @@ -42,7 +42,8 @@ public class Int16Value extends Value public short asInt16() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/Int32Array.java b/jrt/src/com/yahoo/jrt/Int32Array.java index dddcaf83ddc..bc003fa7246 100644 --- a/jrt/src/com/yahoo/jrt/Int32Array.java +++ b/jrt/src/com/yahoo/jrt/Int32Array.java @@ -47,7 +47,8 @@ public class Int32Array extends Value public int[] asInt32Array() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/Int32Value.java b/jrt/src/com/yahoo/jrt/Int32Value.java index 0a57014431a..c835181d344 100644 --- a/jrt/src/com/yahoo/jrt/Int32Value.java +++ b/jrt/src/com/yahoo/jrt/Int32Value.java @@ -41,7 +41,8 @@ public class Int32Value extends Value public int asInt32() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/Int64Array.java b/jrt/src/com/yahoo/jrt/Int64Array.java index 00427b9eff4..49a10aa773c 100644 --- a/jrt/src/com/yahoo/jrt/Int64Array.java +++ b/jrt/src/com/yahoo/jrt/Int64Array.java @@ -47,7 +47,8 @@ public class Int64Array extends Value public long[] asInt64Array() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/Int64Value.java b/jrt/src/com/yahoo/jrt/Int64Value.java index 27c5cd5ff45..cf1de9c9db8 100644 --- a/jrt/src/com/yahoo/jrt/Int64Value.java +++ b/jrt/src/com/yahoo/jrt/Int64Value.java @@ -41,7 +41,8 @@ public class Int64Value extends Value public long asInt64() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/Int8Array.java b/jrt/src/com/yahoo/jrt/Int8Array.java index 4dc18a75d8e..f6347f90b9e 100644 --- a/jrt/src/com/yahoo/jrt/Int8Array.java +++ b/jrt/src/com/yahoo/jrt/Int8Array.java @@ -45,7 +45,8 @@ public class Int8Array extends Value public byte[] asInt8Array() { return value; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(value); } diff --git a/jrt/src/com/yahoo/jrt/Int8Value.java b/jrt/src/com/yahoo/jrt/Int8Value.java index e62460f4bd8..044dbb131f9 100644 --- a/jrt/src/com/yahoo/jrt/Int8Value.java +++ b/jrt/src/com/yahoo/jrt/Int8Value.java @@ -41,7 +41,8 @@ public class Int8Value extends Value public byte asInt8() { return value; } - public @Override String toString() { + @Override + public String toString() { return String.valueOf(value); } diff --git a/jrt/src/com/yahoo/jrt/Method.java b/jrt/src/com/yahoo/jrt/Method.java index d752e33b0d8..7883b20499f 100644 --- a/jrt/src/com/yahoo/jrt/Method.java +++ b/jrt/src/com/yahoo/jrt/Method.java @@ -283,7 +283,8 @@ public class Method { } } - public @Override String toString() { + @Override + public String toString() { return "method " + name + "(" + paramTypes + ")" + ( returnTypes.length()>0 ? ": " + returnTypes : ""); } diff --git a/jrt/src/com/yahoo/jrt/Request.java b/jrt/src/com/yahoo/jrt/Request.java index 4786124e56b..a4ba383653b 100644 --- a/jrt/src/com/yahoo/jrt/Request.java +++ b/jrt/src/com/yahoo/jrt/Request.java @@ -273,7 +273,8 @@ public class Request return errorMessage; } - public @Override String toString() { + @Override + public String toString() { return "request " + methodName + "(" + parameters + ")" + ( returnValues.size()>0 ? ": " + returnValues : ""); } diff --git a/jrt/src/com/yahoo/jrt/StringArray.java b/jrt/src/com/yahoo/jrt/StringArray.java index 8195a586f9d..aef0e052cc7 100644 --- a/jrt/src/com/yahoo/jrt/StringArray.java +++ b/jrt/src/com/yahoo/jrt/StringArray.java @@ -75,7 +75,8 @@ public class StringArray extends Value return ret; } - public @Override String toString() { + @Override + public String toString() { return Arrays.toString(asStringArray()); } diff --git a/jrt/src/com/yahoo/jrt/StringValue.java b/jrt/src/com/yahoo/jrt/StringValue.java index 175884c435b..fe43ec0f609 100644 --- a/jrt/src/com/yahoo/jrt/StringValue.java +++ b/jrt/src/com/yahoo/jrt/StringValue.java @@ -59,9 +59,11 @@ public class StringValue extends Value return value.toString(); } - public @Override Utf8Array asUtf8Array() { return value; } + @Override + public Utf8Array asUtf8Array() { return value; } - public @Override String toString() { + @Override + public String toString() { return asString(); } diff --git a/jrt/src/com/yahoo/jrt/Values.java b/jrt/src/com/yahoo/jrt/Values.java index 05a444e4c6d..2a3213e7dcd 100644 --- a/jrt/src/com/yahoo/jrt/Values.java +++ b/jrt/src/com/yahoo/jrt/Values.java @@ -125,7 +125,8 @@ public class Values } } - public @Override String toString() { + @Override + public String toString() { if (values.size()==0) return ""; if (values.size()==1) return values.get(0).toString(); StringBuffer buffer=new StringBuffer(); diff --git a/juniper/src/test/appender_test.cpp b/juniper/src/test/appender_test.cpp index 7887247d9f2..32a58a16bb0 100644 --- a/juniper/src/test/appender_test.cpp +++ b/juniper/src/test/appender_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #define _NEED_SUMMARY_CONFIG_IMPL diff --git a/juniper/src/test/fakerewriter.cpp b/juniper/src/test/fakerewriter.cpp index c02fc29b119..962f3d0e2f9 100644 --- a/juniper/src/test/fakerewriter.cpp +++ b/juniper/src/test/fakerewriter.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fakerewriter.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/juniper/src/test/matchobjectTest.cpp b/juniper/src/test/matchobjectTest.cpp index 81b3f0e1a79..32f249ae720 100644 --- a/juniper/src/test/matchobjectTest.cpp +++ b/juniper/src/test/matchobjectTest.cpp @@ -16,7 +16,6 @@ * ALL RIGHTS RESERVED * </pre> ***************************************************************************/ -#include <vespa/fastos/fastos.h> #include "matchobjectTest.h" #include "fakerewriter.h" diff --git a/juniper/src/test/queryparserTest.cpp b/juniper/src/test/queryparserTest.cpp index c78af7a98c2..6f1065f279c 100644 --- a/juniper/src/test/queryparserTest.cpp +++ b/juniper/src/test/queryparserTest.cpp @@ -16,7 +16,6 @@ * ALL RIGHTS RESERVED * </pre> ***************************************************************************/ -#include <vespa/fastos/fastos.h> #include "queryparserTest.h" #include "fakerewriter.h" diff --git a/juniper/src/test/queryvisitor_test.cpp b/juniper/src/test/queryvisitor_test.cpp index 18dda67f43f..612336ae78b 100644 --- a/juniper/src/test/queryvisitor_test.cpp +++ b/juniper/src/test/queryvisitor_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <memory> -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/juniper/queryhandle.h> diff --git a/juniper/src/test/testenv.cpp b/juniper/src/test/testenv.cpp index 17187e33a28..ff4331148d3 100644 --- a/juniper/src/test/testenv.cpp +++ b/juniper/src/test/testenv.cpp @@ -3,7 +3,6 @@ * within test framework */ -#include <vespa/fastos/fastos.h> #include "testenv.h" #include <vespa/juniper/propreader.h> diff --git a/juniper/src/vespa/juniper/Matcher.cpp b/juniper/src/vespa/juniper/Matcher.cpp index 2eeac3417af..76460323f9b 100644 --- a/juniper/src/vespa/juniper/Matcher.cpp +++ b/juniper/src/vespa/juniper/Matcher.cpp @@ -6,14 +6,10 @@ #include "juniperdebug.h" #include "sumdesc.h" #include "Matcher.h" -#include "foreach_utils.h" -#include "SummaryConfig.h" -#include "querynode.h" -#include "mcand.h" -#include "matchobject.h" #include "result.h" #include "juniperparams.h" #include "config.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".juniper.matcher"); diff --git a/juniper/src/vespa/juniper/querynode.cpp b/juniper/src/vespa/juniper/querynode.cpp index 8718a76bf61..47bd5146432 100644 --- a/juniper/src/vespa/juniper/querynode.cpp +++ b/juniper/src/vespa/juniper/querynode.cpp @@ -4,6 +4,7 @@ #include "queryvisitor.h" #include "juniperdebug.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.querynode"); @@ -145,12 +146,10 @@ void QueryNode::ComputeThreshold() else if (!(_options & X_AND)) no_threshold = true; - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { QueryExpr* qe = _children[i]; qe->ComputeThreshold(); - if (!no_threshold) - { + if (!no_threshold) { int w = qe->_weight; if (_options | X_AND) th += w; else @@ -232,8 +231,7 @@ bool QueryTerm::Complex() bool QueryNode::Complex() { - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { if (_children[i]->_arity > 1) return true; } return false; @@ -243,8 +241,7 @@ bool QueryNode::Complex() int QueryNode::MaxArity() { int max_arity = _arity; - for (int i = 0; i < _nchild; i++) - { + for (int i = 0; i < _nchild; i++) { int ma = _children[i]->MaxArity(); if (ma > max_arity) max_arity = ma; } @@ -273,8 +270,7 @@ void SimplifyStack(QueryExpr*& orig_stack) int compact = 0; int i; - if (!node->Complete()) - { + if (!node->Complete()) { LOG(warning, "juniper: query stack incomplete, got arity %d, expected %d", node->_nchild, node->_arity); delete node; @@ -282,10 +278,8 @@ void SimplifyStack(QueryExpr*& orig_stack) return; } - for (i = 0; i < node->_arity; i++) - { - if (i > 0 && (node->_options & X_ONLY_1)) - { + for (i = 0; i < node->_arity; i++) { + if (i > 0 && (node->_options & X_ONLY_1)) { // Get rid of children # >2 for RANK/ANDNOT delete node->_children[i]; node->_children[i] = NULL; @@ -296,15 +290,11 @@ void SimplifyStack(QueryExpr*& orig_stack) if (node->_children[i] == NULL) compact++; } - if (compact > 0) - { + if (compact > 0) { node->_nchild = 0; - for (i = 0; i < node->_arity; i++) - { - if (node->_children[i]) - { - if (i > node->_nchild) - { + for (i = 0; i < node->_arity; i++) { + if (node->_children[i]) { + if (i > node->_nchild) { // shift remaining nodes down - remember to update _childno for each node.. node->_children[node->_nchild] = node->_children[i]; node->_children[i]->_childno = node->_nchild; @@ -316,11 +306,9 @@ void SimplifyStack(QueryExpr*& orig_stack) node->_arity = node->_nchild; } - if (node->_arity <= 1) - { + if (node->_arity <= 1) { QueryExpr* ret = NULL; - if (node->_arity == 1) - { + if (node->_arity == 1) { ret = node->_children[0]; node->_children[0] = NULL; ret->_parent = node->_parent; diff --git a/juniper/src/vespa/juniper/queryparser.cpp b/juniper/src/vespa/juniper/queryparser.cpp index 2355e41d005..4acc70dc8ab 100644 --- a/juniper/src/vespa/juniper/queryparser.cpp +++ b/juniper/src/vespa/juniper/queryparser.cpp @@ -5,6 +5,7 @@ #include "queryparser.h" #include "juniperdebug.h" #include <vector> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.queryparser"); diff --git a/juniper/src/vespa/juniper/rpinterface.cpp b/juniper/src/vespa/juniper/rpinterface.cpp index 9f10972a448..890481b4b94 100644 --- a/juniper/src/vespa/juniper/rpinterface.cpp +++ b/juniper/src/vespa/juniper/rpinterface.cpp @@ -1,20 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpinterface.h" -#include <string> -#include <vector> #include "juniperparams.h" #include "foreach_utils.h" -#include "juniperdebug.h" -#include "SummaryConfig.h" #include "queryvisitor.h" -#include "querynode.h" #include "queryhandle.h" #include "propreader.h" #include "result.h" #include "config.h" -#include "querymodifier.h" -#include <vespa/fastlib/text/normwordfolder.h> +#include <vector> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".juniper.rpinterface"); diff --git a/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java b/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java index f4e1ccc9feb..e426b2b6d54 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java @@ -4,7 +4,7 @@ package com.yahoo.language.process; /** * <p>This interface provides NFKC normalization of Strings through the underlying linguistics library.</p> * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00F8lster Lidal</a> + * @author Mathias Mølster Lidal */ public interface Normalizer { @@ -15,5 +15,6 @@ public interface Normalizer { * @return The normalized String. * @throws ProcessingException If underlying library throws an Exception. */ - public String normalize(String input); + String normalize(String input); + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java b/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java index 73764e06ef6..11c00f2163b 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Segmenter.java @@ -6,11 +6,11 @@ import com.yahoo.language.Language; import java.util.List; /** - * <p>Interface providing segmentation, i.e. splitting of CJK character blocks into separate tokens. This is primarily a + * Interface providing segmentation, i.e. splitting of CJK character blocks into separate tokens. This is primarily a * convenience feature for users who don't need full tokenization (or who use a separate tokenizer and only need CJK - * processing).</p> + * processing). * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public interface Segmenter { diff --git a/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java b/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java index 146d65cb7e2..f0db2436037 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java +++ b/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java @@ -7,7 +7,7 @@ import java.util.ArrayList; import java.util.List; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public class SegmenterImpl implements Segmenter { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleNormalizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleNormalizer.java index bfc6f813452..e7459604bee 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleNormalizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleNormalizer.java @@ -4,7 +4,7 @@ package com.yahoo.language.simple; import com.yahoo.language.process.Normalizer; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class SimpleNormalizer implements Normalizer { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 48a12c54e86..a8cd782889c 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -16,7 +16,7 @@ import java.util.List; * * <p>This is not multithread safe.</p> * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal * @author bratseth */ public class SimpleTokenizer implements Tokenizer { diff --git a/logd/src/logd/forward.cpp b/logd/src/logd/forward.cpp index 6e980034c9b..560ccaf1063 100644 --- a/logd/src/logd/forward.cpp +++ b/logd/src/logd/forward.cpp @@ -3,10 +3,10 @@ #include "forward.h" #include "errhandle.h" #include <vespa/vespalib/component/vtag.h> +#include <unistd.h> #include <vespa/log/log.h> LOG_SETUP(""); -LOG_RCSID("$Id$"); namespace logdemon { diff --git a/logd/src/logd/service.h b/logd/src/logd/service.h index 700ab85fe27..3ff2e7fab92 100644 --- a/logd/src/logd/service.h +++ b/logd/src/logd/service.h @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <assert.h> + #include <logd/config-logd.h> #include <vespa/vespalib/util/hashmap.h> #include <vespa/log/log.h> +#include <cassert> namespace logdemon { diff --git a/logd/src/tests/info/info.cpp b/logd/src/tests/info/info.cpp index c674bd74d88..a54fd261801 100644 --- a/logd/src/tests/info/info.cpp +++ b/logd/src/tests/info/info.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("info_test"); -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> TEST_SETUP(Test); diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index 6d215b9e831..2b49abd1ff4 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -10,37 +10,37 @@ import java.util.logging.Logger; /** * TODO: describe class * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ public abstract class AbstractPluginLoader implements PluginLoader { - private static final Logger log = Logger.getLogger(AbstractPluginLoader.class.getName()); + private static final Logger log = Logger.getLogger(AbstractPluginLoader.class.getName()); - public abstract void loadPlugins(); + public abstract void loadPlugins(); - protected void loadFromClass (Class<? extends Plugin> pluginClass) { - Plugin plugin; - try { - plugin = (Plugin)pluginClass.newInstance(); - } catch (InstantiationException | IllegalAccessException e) { - log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e); - throw new RuntimeException(e); - } + protected void loadFromClass(Class<? extends Plugin> pluginClass) { + Plugin plugin; + try { + plugin = (Plugin) pluginClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e); + throw new RuntimeException(e); + } String pname = plugin.getPluginName(); - String prefix = Server.APPNAME + "." + pname + "."; - SystemPropertyConfig config = new SystemPropertyConfig(prefix); - String enable = config.get("enable", "true"); - - if (!enable.equals("true")) { - log.log(LogLevel.INFO, pname + ": plugin disabled by config"); - return; - } - - try { - plugin.initPlugin(config); - log.log(LogLevel.DEBUG, pname + ": plugin loaded"); - } catch (Exception e) { + String prefix = Server.APPNAME + "." + pname + "."; + SystemPropertyConfig config = new SystemPropertyConfig(prefix); + String enable = config.get("enable", "true"); + + if (! enable.equals("true")) { + log.log(LogLevel.INFO, pname + ": plugin disabled by config"); + return; + } + + try { + plugin.initPlugin(config); + log.log(LogLevel.DEBUG, pname + ": plugin loaded"); + } catch (Exception e) { log.log(LogLevel.ERROR, pname + ": init failed", e); } - } + } } diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index f2ef60267fe..5f8230bd246 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -12,7 +12,7 @@ import com.yahoo.logserver.handlers.replicator.ReplicatorPlugin; /** * Load a set of builtin plugins * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ public class BuiltinPluginLoader extends AbstractPluginLoader { private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName()); diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index ee2b317f573..19b06c15f85 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -11,7 +11,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.logserver.handlers.LogHandler; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Flusher extends Thread { private static final Logger log = Logger.getLogger(Flusher.class.getName()); @@ -29,7 +29,7 @@ public class Flusher extends Thread { } public static synchronized void register(LogHandler logHandler) { - logHandlers.add(new WeakReference<LogHandler>(logHandler)); + logHandlers.add(new WeakReference<>(logHandler)); } public synchronized void run() { diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index 6de34d4a899..e13a2f09323 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -15,24 +15,22 @@ import com.yahoo.logserver.handlers.LogHandler; * This is the central point from which LogMessage objects are * propagated throughout the logserver architecture. * - * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ -public class LogDispatcher implements LogHandler, SelectLoopHook -{ +public class LogDispatcher implements LogHandler, SelectLoopHook { private static final Logger log = Logger.getLogger(LogDispatcher.class.getName()); - private final List<LogHandler> handlers = new ArrayList<LogHandler>(); - private int messageCount = 0; + private final List<LogHandler> handlers = new ArrayList<>(); + private int messageCount = 0; private boolean hasBeenShutDown = false; private boolean batchedMode = false; private final int batchSize = 5000; private List<LogMessage> currentBatchList; private int roundCount = 0; @SuppressWarnings("unused") - private int lastRoundCount = 0; + private int lastRoundCount = 0; - public LogDispatcher () { + public LogDispatcher() { } /** @@ -43,7 +41,7 @@ public class LogDispatcher implements LogHandler, SelectLoopHook * @param msg The LogMessage instance we wish to dispatch to the * plugins */ - public synchronized void handle (LogMessage msg) { + public synchronized void handle(LogMessage msg) { if (msg == null) { throw new NullPointerException("LogMessage was null"); } @@ -51,14 +49,14 @@ public class LogDispatcher implements LogHandler, SelectLoopHook if (batchedMode) { addToBatch(msg); } else { - for (LogHandler h : handlers) { + for (LogHandler h : handlers) { h.handle(msg); } } messageCount++; } - private void addToBatch (LogMessage msg) { + private void addToBatch(LogMessage msg) { if (currentBatchList == null) { currentBatchList = new ArrayList<LogMessage>(batchSize); currentBatchList.add(msg); @@ -72,19 +70,19 @@ public class LogDispatcher implements LogHandler, SelectLoopHook } } - private void flushBatch () { - if (currentBatchList == null) { - return; + private void flushBatch() { + List<LogMessage> todo; + synchronized(this) { + todo = currentBatchList; + currentBatchList = null; } - + if (todo == null) return; for (LogHandler ht : handlers) { - ht.handle(currentBatchList); + ht.handle(todo); } - currentBatchList = null; } - - public void handle (List<LogMessage> messages) { + public void handle(List<LogMessage> messages) { throw new IllegalStateException("method not supported"); } @@ -95,11 +93,11 @@ public class LogDispatcher implements LogHandler, SelectLoopHook * dispatcher will not enqueue single LogMessage instances * but lists of same. */ - public void setBatchedMode (boolean batchedMode) { + public void setBatchedMode(boolean batchedMode) { this.batchedMode = batchedMode; } - public synchronized void flush () { + public synchronized void flush() { if (batchedMode) { flushBatch(); } @@ -112,7 +110,7 @@ public class LogDispatcher implements LogHandler, SelectLoopHook } } - public synchronized void close () { + public synchronized void close() { if (hasBeenShutDown) { throw new IllegalStateException("Shutdown already in progress"); } @@ -121,7 +119,7 @@ public class LogDispatcher implements LogHandler, SelectLoopHook for (LogHandler ht : handlers) { if (ht instanceof Thread) { log.fine("Stopping " + ht); - ((Thread)ht).interrupt(); + ((Thread) ht).interrupt(); } } handlers.clear(); @@ -133,15 +131,13 @@ public class LogDispatcher implements LogHandler, SelectLoopHook * Register handler thread with the dispatcher. If the handler * thread has already been registered, we log a warning and * just do nothing. - * - * <P> + * <p> * If the thread is not alive it will be start()'ed. - * */ - public synchronized void registerLogHandler (LogHandler ht) { + public synchronized void registerLogHandler(LogHandler ht) { if (hasBeenShutDown) { - throw new IllegalStateException("Tried to register LogHandler on"+ - " LogDispatcher which was shut down"); + throw new IllegalStateException("Tried to register LogHandler on" + + " LogDispatcher which was shut down"); } if (handlers.contains(ht)) { @@ -150,8 +146,8 @@ public class LogDispatcher implements LogHandler, SelectLoopHook } handlers.add(ht); - if ((ht instanceof Thread) && (! ((Thread)ht).isAlive())) { - ((Thread)ht).start(); + if ((ht instanceof Thread) && (! ((Thread) ht).isAlive())) { + ((Thread) ht).start(); } log.fine("Added (and possibly started) LogHandler " + ht); @@ -160,7 +156,7 @@ public class LogDispatcher implements LogHandler, SelectLoopHook /** * Make defensive copy and return array of LogHandlers. */ - public LogHandler[] getLogHandlers () { + public LogHandler[] getLogHandlers() { LogHandler[] h = new LogHandler[handlers.size()]; return handlers.toArray(h); } @@ -170,14 +166,14 @@ public class LogDispatcher implements LogHandler, SelectLoopHook * * @return Returns the number of messages that we have seen. */ - public synchronized int getMessageCount () { + public synchronized int getMessageCount() { return messageCount; } /** * Hook which is called when the select loop has finished. */ - public void selectLoopHook (boolean before) { + public void selectLoopHook(boolean before) { if (batchedMode) { flushBatch(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java index 2357eefb804..42355022f16 100644 --- a/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/PluginLoader.java @@ -7,16 +7,14 @@ package com.yahoo.logserver; * all of its knows plugins. In addition, if a plugin loader's * canReload() method returns <code>true</code>, plugins may be loaded * again after they are unloaded. - * * <p> Plugins loaded through such reload-capable plugin loaders may * be upgraded without restarting the server. * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ -public interface PluginLoader -{ +public interface PluginLoader { /** * Load all plugins known to this loader. */ - public void loadPlugins(); + void loadPlugins(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index d09ae305dc2..dbfd7e028c1 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -11,6 +11,7 @@ import com.yahoo.logserver.handlers.LogHandler; import com.yahoo.logserver.net.LogConnectionFactory; import com.yahoo.logserver.net.control.Levels; import com.yahoo.system.CatchSigTerm; + import java.io.IOException; import java.util.HashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -22,21 +23,18 @@ import java.util.logging.Logger; * no runtime configuration; the server starts up, binds port 19081 * and loads builtin plugins. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Bjorn Borud + * @author Stig Bakken */ public class Server implements Runnable { private final AtomicBoolean signalCaught = new AtomicBoolean(false); - public static final String APPNAME = "logserver"; + static final String APPNAME = "logserver"; private static final Server instance = new Server(); private static final Logger log = Logger.getLogger(Server.class.getName()); - private static final FatalErrorHandler fatalErrorHandler = - new FatalErrorHandler(); - private static final HashMap<String,HandlerThread> handlerThreads = - new HashMap<String,HandlerThread>(); - private static final HashMap<LogHandler,String> threadNameForHandler = - new HashMap<LogHandler,String>(); + private static final FatalErrorHandler fatalErrorHandler = new FatalErrorHandler(); + private static final HashMap<String, HandlerThread> handlerThreads = new HashMap<>(); + private static final HashMap<LogHandler, String> threadNameForHandler = new HashMap<>(); static { LogSetup.initVespaLogging("ADM"); @@ -44,7 +42,7 @@ public class Server implements Runnable { // the port is a String because we want to use it as the default // value of a System.getProperty(). - private static final String LISTEN_PORT = "19081"; + private static final String LISTEN_PORT = "19081"; private int listenPort; private Listener listener; @@ -55,17 +53,17 @@ public class Server implements Runnable { /** * Server constructor */ - private Server () { + private Server() { dispatch = new LogDispatcher(); dispatch.setBatchedMode(true); isInitialized = false; } - public static Server getInstance () { + public static Server getInstance() { return instance; } - private HandlerThread getHandlerThread (String threadName) { + private HandlerThread getHandlerThread(String threadName) { threadName += " handler thread"; HandlerThread ht = handlerThreads.get(threadName); if (ht == null) { @@ -77,33 +75,35 @@ public class Server implements Runnable { return ht; } - public void registerPluginLoader (PluginLoader loader) { + public void registerPluginLoader(PluginLoader loader) { loader.loadPlugins(); } - public void registerLogHandler (LogHandler lh, String threadName) { + public void registerLogHandler(LogHandler lh, String threadName) { HandlerThread ht = getHandlerThread(threadName); ht.registerHandler(lh); threadNameForHandler.put(lh, threadName); } - public void unregisterLogHandler (LogHandler lh) { + public void unregisterLogHandler(LogHandler lh) { String threadName = threadNameForHandler.get(lh); unregisterLogHandler(lh, threadName); } - public void unregisterLogHandler (LogHandler lh, String threadName) { + public void unregisterLogHandler(LogHandler lh, String threadName) { HandlerThread ht = getHandlerThread(threadName); ht.unregisterHandler(lh); threadNameForHandler.remove(lh); } - public void registerFlusher (LogHandler lh) { + public void registerFlusher(LogHandler lh) { Flusher.register(lh); } - /** Included only for consistency */ - public void unregisterFlusher (LogHandler lh) { + /** + * Included only for consistency + */ + public void unregisterFlusher(LogHandler lh) { /* NOP */ } @@ -113,7 +113,7 @@ public class Server implements Runnable { * @param listenPort The port on which the logserver accepts log * messages. */ - public void initialize (int listenPort) { + public void initialize(int listenPort) { if (isInitialized) { throw new IllegalStateException(APPNAME + " already initialized"); } @@ -148,18 +148,18 @@ public class Server implements Runnable { try { listener.join(); log.fine("listener thread exited"); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { log.log(Level.WARNING, "Server was interrupted", e); } } + private void setupSigTermHandler() { CatchSigTerm.setup(signalCaught); // catch termination signal } private void waitForShutdown() { synchronized (signalCaught) { - while (!signalCaught.get()) { + while (! signalCaught.get()) { try { signalCaught.wait(); } catch (InterruptedException e) { @@ -173,24 +173,20 @@ public class Server implements Runnable { System.exit(0); } - public static HashMap<LogHandler, String> threadNameForHandler() { + static HashMap<LogHandler, String> threadNameForHandler() { return threadNameForHandler; } - - public static void help () { + + static void help() { System.out.println(); System.out.println("System properties:"); System.out.println(" - " + APPNAME + ".listenport (" + LISTEN_PORT + ")"); - System.out.println(" - " + APPNAME + ".queue.size (" - + HandlerThread.DEFAULT_QUEUESIZE - + ")"); - System.out.println(" - logserver.default.loglevels (" - + (new Levels()).toString() - + ")"); + System.out.println(" - " + APPNAME + ".queue.size (" + HandlerThread.DEFAULT_QUEUESIZE + ")"); + System.out.println(" - logserver.default.loglevels (" + (new Levels()).toString() + ")"); System.out.println(); } - public static void main (String[] args) { + public static void main(String[] args) { if (args.length > 0 && "-help".equals(args[0])) { help(); System.exit(0); diff --git a/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore b/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/demo/.gitignore +++ /dev/null diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java index b229af522ab..1d4a817b4c4 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LevelFilter.java @@ -2,29 +2,30 @@ package com.yahoo.logserver.filter; import com.yahoo.log.LogMessage; + import java.util.Set; import java.util.HashSet; import java.util.logging.Level; + /** - * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LevelFilter implements LogFilter { private final Set<Level> levels = new HashSet<Level>(); - public void addLevel (Level level) { + public void addLevel(Level level) { levels.add(level); } - public void removeLevel (Level level) { + public void removeLevel(Level level) { levels.remove(level); } - public boolean isLoggable (LogMessage msg) { + public boolean isLoggable(LogMessage msg) { return levels.contains(msg.getLevel()); } - public String description () { + public String description() { return "Match specific log levels"; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java index 8c4c3911c9e..51bd9adc03e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilter.java @@ -9,15 +9,15 @@ import com.yahoo.log.LogMessage; * <b>stateless/immutable if possible so filters can be * shared</b>. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public interface LogFilter { /** * Determine if this log message is loggable. * * @param msg The log message - * */ - public boolean isLoggable (LogMessage msg); - public String description (); + public boolean isLoggable(LogMessage msg); + + public String description(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java index a68aff86e10..d608c7871f0 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -12,9 +12,8 @@ import com.yahoo.log.LogLevel; * is truly global. It also manages the LogFilter namespace * to ensure that system-defined filters are not tampered with. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ - public class LogFilterManager { private static final LogFilterManager instance; @@ -30,20 +29,20 @@ public class LogFilterManager { instance.addLogFilterInternal("system.mute", MuteFilter.getInstance()); } - private final Map<String,LogFilter> filters = new HashMap<String,LogFilter>(); + private final Map<String, LogFilter> filters = new HashMap<String, LogFilter>(); - private LogFilterManager () { + private LogFilterManager() { } /** * Public interface for adding a name-logfilter mapping. If * there exists a mapping already the old mapping is replaced * with the new mapping. - * + * <p> * If the name is within the namespace reserved for internal * built-in filters it will throw an exception */ - public static void addLogFilter (String name, LogFilter filter) { + public static void addLogFilter(String name, LogFilter filter) { if (filter == null) { throw new NullPointerException("filter cannot be null"); } @@ -66,9 +65,9 @@ public class LogFilterManager { * * @param name The name of the LogFilter to be looked up. * @return Returns the LogFilter associated with this name or - * <code>null</code> if not found. + * <code>null</code> if not found. */ - public static LogFilter getLogFilter (String name) { + public static LogFilter getLogFilter(String name) { return instance.filters.get(name); } @@ -77,12 +76,11 @@ public class LogFilterManager { * Get the names of the defined filters. * * @return Returns an array containing the names of filters that - * have been registered. - * + * have been registered. */ - public static String[] getFilterNames () { - synchronized(instance.filters) { - String[] filterNames= new String[instance.filters.keySet().size()]; + public static String[] getFilterNames() { + synchronized (instance.filters) { + String[] filterNames = new String[instance.filters.keySet().size()]; instance.filters.keySet().toArray(filterNames); return filterNames; } @@ -93,7 +91,7 @@ public class LogFilterManager { * LogFilter mappings but doesn't perform any of the checks * performed by the public method for adding mappings. */ - private void addLogFilterInternal (String name, LogFilter filter) { + private void addLogFilterInternal(String name, LogFilter filter) { filters.put(name, filter); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java index fa626de1d08..3f9ae20dc3b 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java @@ -15,7 +15,7 @@ import com.yahoo.log.LogMessage; * This filter matches events that are used for monitoring, specificly * the Count and Value events. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class MetricsFilter implements LogFilter { public boolean isLoggable (LogMessage msg) { diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java index a360cba6b8f..45071b7344c 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MuteFilter.java @@ -6,7 +6,7 @@ import com.yahoo.log.LogMessage; /** * Filter which always returns false. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class MuteFilter implements LogFilter { private static final MuteFilter instance = new MuteFilter(); @@ -14,17 +14,17 @@ public class MuteFilter implements LogFilter { /** * Singleton, private constructor. */ - private MuteFilter () {} + private MuteFilter() {} public static MuteFilter getInstance() { return instance; } - public boolean isLoggable (LogMessage msg) { + public boolean isLoggable(LogMessage msg) { return false; } - public String description () { + public String description() { return "Matches no messages. Mute."; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java index 5621ea60d0c..4185f318ced 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/NoMetricsFilter.java @@ -6,10 +6,10 @@ import com.yahoo.log.LogMessage; /** * This filter is the complement of MetricsFilter * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class NoMetricsFilter implements LogFilter { - final MetricsFilter filter = new MetricsFilter(); + private final MetricsFilter filter = new MetricsFilter(); public boolean isLoggable (LogMessage msg) { return (! filter.isLoggable(msg)); diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java index a62b2f7171b..9698a02f4f8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/NullFilter.java @@ -4,16 +4,14 @@ package com.yahoo.logserver.filter; import com.yahoo.log.LogMessage; /** - * - * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class NullFilter implements LogFilter { - public boolean isLoggable (LogMessage msg) { + public boolean isLoggable(LogMessage msg) { return true; } - public String description () { + public String description() { return "Match all log messages"; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java index ae217634e1c..ef42a915e24 100644 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatter.java @@ -10,19 +10,18 @@ import com.yahoo.log.LogMessage; * shared</b>. If it does have state it must not prevent * concurrent use. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public interface LogFormatter { /** * Format log message as a string. * * @param msg The log message - * */ - public String format (LogMessage msg); + String format(LogMessage msg); /** * Returns a textual description of the formatter */ - public String description (); + String description(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java index fb81b35dcf0..7fe6c6d17d5 100644 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/LogFormatterManager.java @@ -13,30 +13,30 @@ import java.util.Map; * This singleton class implements a central registry of LogFormatter * instances. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogFormatterManager { private static final LogFormatterManager instance; + static { instance = new LogFormatterManager(); instance.addLogFormatterInternal("system.textformatter", new TextFormatter()); instance.addLogFormatterInternal("system.nullformatter", new NullFormatter()); } - private final Map<String,LogFormatter> logFormatters = - new HashMap<String,LogFormatter>(); + private final Map<String, LogFormatter> logFormatters = new HashMap<String, LogFormatter>(); - private LogFormatterManager () {} + private LogFormatterManager() {} /** * LogFormatter lookup function * * @param name The name of the LogFormatter to be looked up. * @return Returns the LogFormatter associated with this name or - * <code>null</code> if not found. + * <code>null</code> if not found. */ - public static LogFormatter getLogFormatter (String name) { - synchronized(instance.logFormatters) { + public static LogFormatter getLogFormatter(String name) { + synchronized (instance.logFormatters) { return instance.logFormatters.get(name); } } @@ -45,11 +45,10 @@ public class LogFormatterManager { * Get the names of the defined formatters. * * @return Returns an array containing the names of formatters that - * have been registered. - * + * have been registered. */ - public static String[] getFormatterNames () { - synchronized(instance.logFormatters) { + public static String[] getFormatterNames() { + synchronized (instance.logFormatters) { String[] formatterNames = new String[instance.logFormatters.keySet().size()]; instance.logFormatters.keySet().toArray(formatterNames); return formatterNames; @@ -61,8 +60,8 @@ public class LogFormatterManager { * LogFormatter mappings but doesn't perform any of the checks * performed by the public method for adding mappings. */ - private void addLogFormatterInternal (String name, LogFormatter logFormatter) { - synchronized(logFormatters) { + private void addLogFormatterInternal(String name, LogFormatter logFormatter) { + synchronized (logFormatters) { logFormatters.put(name, logFormatter); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java index 646ab0ce423..4bff6454239 100644 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/NullFormatter.java @@ -14,11 +14,11 @@ import com.yahoo.log.LogMessage; * pointless and silly, but we include it for symmetry... * or completeness....or...whatever. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class NullFormatter implements LogFormatter { - public String format (LogMessage msg) { + public String format(LogMessage msg) { return msg.toString(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java index d94dcceaf29..ff1a4742e67 100644 --- a/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java +++ b/logserver/src/main/java/com/yahoo/logserver/formatter/TextFormatter.java @@ -15,7 +15,7 @@ import com.yahoo.log.LogMessage; /** * Creates human-readable text representation of log message. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class TextFormatter implements LogFormatter { static final SimpleDateFormat dateFormat; @@ -25,7 +25,7 @@ public class TextFormatter implements LogFormatter { dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } - public String format (LogMessage msg) { + public String format(LogMessage msg) { StringBuffer sbuf = new StringBuffer(150); sbuf.append(dateFormat.format(new Date(msg.getTime()))) .append(" ") diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java index f498568e7f2..362312434d3 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/AbstractLogHandler.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers; -import java.io.PrintWriter; import java.util.Iterator; import java.util.List; @@ -14,11 +13,11 @@ import com.yahoo.logserver.filter.LogFilter; * of quite a bit of tedious work for you (log message counting, * handling of lists of messages versus single instances etc). * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public abstract class AbstractLogHandler implements LogHandler { - private long count = 0; - private long filtered = 0; + private long count = 0; + private long filtered = 0; private LogFilter filter = null; private String name; @@ -29,7 +28,7 @@ public abstract class AbstractLogHandler implements LogHandler { * * @param msg The message we are about to handle */ - public final void handle (LogMessage msg) { + public final void handle(LogMessage msg) { if ((filter != null) && (! filter.isLoggable(msg))) { filtered++; return; @@ -45,7 +44,7 @@ public abstract class AbstractLogHandler implements LogHandler { * * @param messages List of LogMessage instances. */ - public final void handle (List<LogMessage> messages) { + public final void handle(List<LogMessage> messages) { Iterator<LogMessage> it = messages.iterator(); while (it.hasNext()) { handle(it.next()); @@ -58,17 +57,16 @@ public abstract class AbstractLogHandler implements LogHandler { * off. * * @param filter The filter to be used for this handler - * */ - public void setLogFilter (LogFilter filter) { + public void setLogFilter(LogFilter filter) { this.filter = filter; } /** * @return Returns the log filter for this handler or - * <code>null</code> if no filter is in effect. + * <code>null</code> if no filter is in effect. */ - public LogFilter getLogFilter () { + public LogFilter getLogFilter() { return filter; } @@ -78,23 +76,23 @@ public abstract class AbstractLogHandler implements LogHandler { * * @return Returns the number of times doHandle has been called. */ - public final long getCount () { + public final long getCount() { return count; } - public String getName () { + public String getName() { if (name == null) { String n = this.getClass().getName(); int x = n.lastIndexOf('.'); - if (x != -1) { - n = n.substring(x+1); + if (x != - 1) { + n = n.substring(x + 1); } name = n; } return name; } - public void setName (String name) { + public void setName(String name) { this.name = name; } @@ -102,8 +100,7 @@ public abstract class AbstractLogHandler implements LogHandler { * The method which actually handles the log message and * does something to it. This is the one you wish to * override when you write a new handler. - * - * <P> + * <p> * <em> * If your handle method is slow you should document this fact * so that decisions can be made with regard to configuration. @@ -111,24 +108,23 @@ public abstract class AbstractLogHandler implements LogHandler { * * @param msg The LogMessage we are about to handle * @return Returns <code>true</code> if the message was - * handled and <code>false</code> if it was ignored. + * handled and <code>false</code> if it was ignored. */ - public abstract boolean doHandle (LogMessage msg); + public abstract boolean doHandle(LogMessage msg); /** * Flush LogMessages. */ - public abstract void flush (); + public abstract void flush(); /** * Close this loghandler. After a loghandler is closed calling * the #handle() has undefined behavior, but it should be assumed * that log messages will be silently dropped. - * - * <P> + * <p> * #close() usually implies #flush() but don't bet on it. */ - public abstract void close (); + public abstract void close(); /** * Force implementation of (hopefully meaningful) toString() diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java index 15c44d6e197..d6e8f30298a 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -18,11 +18,10 @@ import java.util.concurrent.LinkedBlockingQueue; * thread. The purpose of this handler is to isolate execution * of handlers from the main server IO threads. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ -public class HandlerThread extends Thread implements LogHandler -{ +public class HandlerThread extends Thread implements LogHandler { private static final Logger log = Logger.getLogger(HandlerThread.class.getName()); // default queue size is 200 @@ -49,16 +48,19 @@ public class HandlerThread extends Thread implements LogHandler private static class ItemOrList { final LogMessage item; final List<LogMessage> list; + ItemOrList(LogMessage i) { this.item = i; this.list = null; } + ItemOrList(List<LogMessage> l) { this.item = null; this.list = l; } + public String toString() { - return "item="+item+", list="+list; + return "item=" + item + ", list=" + list; } } @@ -66,15 +68,15 @@ public class HandlerThread extends Thread implements LogHandler private final List<LogHandler> handlers = new ArrayList<LogHandler>(); private long count; @SuppressWarnings("unused") - private long droppedCount = 0; + private long droppedCount = 0; @SuppressWarnings("unused") - private boolean queueWasFull = false; + private boolean queueWasFull = false; @SuppressWarnings("unused") - private long lastDropLogMessage = 0; + private long lastDropLogMessage = 0; @SuppressWarnings("unused") - private long lastAcceptingLogMessage = 0; + private long lastAcceptingLogMessage = 0; - public HandlerThread (String name) { + public HandlerThread(String name) { super(name); queue = new LinkedBlockingQueue<>(queueSize); log.log(LogLevel.CONFIG, "logserver.queue.size=" + queueSize); @@ -85,7 +87,7 @@ public class HandlerThread extends Thread implements LogHandler * * @param f The FatalErrorHandler instance to be registered */ - public synchronized void setFatalErrorHandler (FatalErrorHandler f) { + public synchronized void setFatalErrorHandler(FatalErrorHandler f) { fatalErrorHandler = f; } @@ -95,7 +97,7 @@ public class HandlerThread extends Thread implements LogHandler * @param message The LogMessage we wish to dispatch to this * handler thread. */ - public void handle (LogMessage message) { + public void handle(LogMessage message) { handleInternal(new ItemOrList(message)); } @@ -103,24 +105,23 @@ public class HandlerThread extends Thread implements LogHandler * Called by the LogDispatch to put a list of LogMessage * instances onto the Queue. */ - public void handle (List<LogMessage> messages) { + public void handle(List<LogMessage> messages) { handleInternal(new ItemOrList(messages)); } - private void handleInternal (ItemOrList o) { + private void handleInternal(ItemOrList o) { boolean done = false; while (! done) { try { queue.put(o); done = true; - } - catch (InterruptedException e) { + } catch (InterruptedException e) { // NOP } } } - public void flush () { + public void flush() { Iterator<LogHandler> it = handlers.iterator(); while (it.hasNext()) { LogHandler handler = it.next(); @@ -128,7 +129,7 @@ public class HandlerThread extends Thread implements LogHandler } } - public void close () { + public void close() { Iterator<LogHandler> it = handlers.iterator(); while (it.hasNext()) { LogHandler handler = it.next(); @@ -136,14 +137,14 @@ public class HandlerThread extends Thread implements LogHandler } } - public long getCount () { + public long getCount() { return count; } /** * Register a LogHandler */ - public synchronized void registerHandler (LogHandler handler) { + public synchronized void registerHandler(LogHandler handler) { log.fine("Registering handler " + handler); handlers.add(handler); } @@ -151,15 +152,15 @@ public class HandlerThread extends Thread implements LogHandler /** * Unregister a Loghandler */ - public synchronized void unregisterHandler (LogHandler handler) { - int idx; - while ((idx = handlers.indexOf(handler)) != -1) { - try { - handlers.remove(idx); - } catch (IndexOutOfBoundsException e) { - throw new RuntimeException(e); - } - } + public synchronized void unregisterHandler(LogHandler handler) { + int idx; + while ((idx = handlers.indexOf(handler)) != - 1) { + try { + handlers.remove(idx); + } catch (IndexOutOfBoundsException e) { + throw new RuntimeException(e); + } + } } /** @@ -167,7 +168,7 @@ public class HandlerThread extends Thread implements LogHandler * * @return Returns an array of the handlers registered */ - public LogHandler[] getHandlers () { + public LogHandler[] getHandlers() { LogHandler[] h = new LogHandler[handlers.size()]; return handlers.toArray(h); } @@ -177,7 +178,7 @@ public class HandlerThread extends Thread implements LogHandler * Return the underlying queue used to send LogMessage instances * to this handler thread. */ - public BlockingQueue<ItemOrList> getQueue () { + public BlockingQueue<ItemOrList> getQueue() { return queue; } @@ -185,15 +186,15 @@ public class HandlerThread extends Thread implements LogHandler * Consume messages from the incoming queue and hand * them off to the handlers. */ - public void run () { + public void run() { if (queue == null) { throw new NullPointerException("channel is not allowed to be null"); } - // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly. + // TODO: Make the legmessage elements some kind of composite structure to handle both individual messages and lists uniformly. List<ItemOrList> drainList = new ArrayList<ItemOrList>(queue.size() + 1); try { - for (;;) { + for (; ; ) { drainList.clear(); // block in take(), then see if there is more // to be had with drainTo() @@ -206,15 +207,15 @@ public class HandlerThread extends Thread implements LogHandler // handle them accordingly. if (o.item != null) { - for (LogHandler handler : handlers) { + for (LogHandler handler : handlers) { handler.handle(o.item); } } else if (o.list != null) { - for (LogHandler handler : handlers) { + for (LogHandler handler : handlers) { handler.handle(o.list); } } else { - throw new IllegalArgumentException("not LogMessage or List: " + o); + throw new IllegalArgumentException("not LogMessage or List: " + o); } count++; } @@ -227,8 +228,8 @@ public class HandlerThread extends Thread implements LogHandler } } finally { log.fine("Handler thread " - + getName() - + " exiting, removing handlers"); + + getName() + + " exiting, removing handlers"); for (LogHandler handler : handlers) { log.fine("Removing handler " + handler); handler.close(); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java index 6fb009fcda2..c613a04b75b 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/LogHandler.java @@ -2,13 +2,14 @@ package com.yahoo.logserver.handlers; import com.yahoo.log.LogMessage; + import java.util.List; /** * The LogHandler interface defines the interface used for all * parts of the logserver which consume log messages. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public interface LogHandler { /** @@ -19,7 +20,7 @@ public interface LogHandler { * * @param msg The log message */ - public void handle (LogMessage msg); + void handle(LogMessage msg); /** * Instead of taking a single log message, this method can take @@ -27,25 +28,24 @@ public interface LogHandler { * order needs to be preserved. * * @param messages a List containing zero or more LogMessage - * instances. + * instances. */ - public void handle (List<LogMessage> messages); + void handle(List<LogMessage> messages); /** * Any log messages received so far should be dealt with * before this method returns -- within reason ,of course. * (<em>Within reason is loosely defined to be 2-5 seconds</em>) */ - public void flush (); + void flush(); /** * Signals that we want to end logging and should close down the - * unerlying logging mechanism -- whatever this maps to + * underlying logging mechanism -- whatever this maps to * semantically for the underlying implementation. After this * method has been called it is considered an error to submit more * log messages to the handle() methods and an implementation * may elect to throw runtime exceptions. - * */ - public void close (); + void close(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index 06bf5183005..b1169aff02e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -24,56 +24,63 @@ import com.yahoo.logserver.handlers.AbstractLogHandler; * messages based on their timestamp. The goal of this archiver * is to make it easy to locate messages in a time interval, while * ensuring that no log file exceeds the maximum allowed size. - * * <p> * This class is not thread safe. * </p> - * * <p> * TODO: * </p> * <ul> - * <li> Add file locking support in order to make it - * possible to do concurrent compression of log - * files. - * - * <li> Add support for disk monitoring. Should have - * high/low watermark mechanism and three modes - * of operation: normal, tight and full. In - * "tight" mode disk is running low and compression - * and cleanup should possibly be more frequent. - * - * <li> Add compression task which periodically scans - * the log directory looking for uncompressed - * candidate log files. - * + * <li> Add file locking support in order to make it + * possible to do concurrent compression of log + * files. + * </li> + * <li> Add support for disk monitoring. Should have + * high/low watermark mechanism and three modes + * of operation: normal, tight and full. In + * "tight" mode disk is running low and compression + * and cleanup should possibly be more frequent. + * </li> + * <li> Add compression task which periodically scans + * the log directory looking for uncompressed + * candidate log files. + * </li> * </ul> - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + * @author Bjorn Borud */ -public class ArchiverHandler extends AbstractLogHandler -{ - private static final Logger log - = Logger.getLogger(ArchiverHandler.class.getName()); +public class ArchiverHandler extends AbstractLogHandler { + private static final Logger log = Logger.getLogger(ArchiverHandler.class.getName()); - /** File instance representing root directory for logging */ + /** + * File instance representing root directory for logging + */ private File root; - /** Root directory for logging */ + /** + * Root directory for logging + */ private String absoluteRootDir; - /** Max number of log files open at any given time */ + /** + * Max number of log files open at any given time + */ private final int maxFilesOpen = 100; /** - *The maximum number of bytes we allow a file to grow to + * The maximum number of bytes we allow a file to grow to * before we rotate it */ private int maxFileSize; - /** Calendar instance for operating on Date objects */ + /** + * Calendar instance for operating on Date objects + */ private final Calendar calendar; - /** DateFormat instance for building filenames */ + /** + * DateFormat instance for building filenames + */ private final SimpleDateFormat dateformat; /** @@ -91,14 +98,14 @@ public class ArchiverHandler extends AbstractLogHandler * Creates an ArchiverHandler which puts files under * the given root directory. */ - public ArchiverHandler () { + public ArchiverHandler() { calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); dateformat = new SimpleDateFormat("yyyy/MM/dd/HH"); dateformat.setTimeZone(TimeZone.getTimeZone("UTC")); // Set up filtering String archiveMetrics = System.getProperty("vespa_log_server__archive_metric"); - if ("off".equals(archiveMetrics)){ + if ("off".equals(archiveMetrics)) { filter = LogFilterManager.getLogFilter("system.nometricsevents"); } @@ -106,14 +113,14 @@ public class ArchiverHandler extends AbstractLogHandler // set up LRU for files logWriterLRUCache = new LogWriterLRUCache(maxFilesOpen, - (float)0.75); + (float) 0.75); } /** * Creates an ArchiverHandler which puts files under * the given root directory. */ - public ArchiverHandler (String rootDir, int maxFileSize) { + public ArchiverHandler(String rootDir, int maxFileSize) { this(); setRootDir(rootDir); this.maxFileSize = maxFileSize; @@ -123,8 +130,8 @@ public class ArchiverHandler extends AbstractLogHandler /** * Return the appropriate LogWriter given a log message. */ - private synchronized LogWriter getLogWriter (LogMessage m) throws IOException { - Integer slot = new Integer(dateHash(m.getTime())); + private synchronized LogWriter getLogWriter(LogMessage m) throws IOException { + Integer slot = dateHash(m.getTime()); LogWriter logWriter = logWriterLRUCache.get(slot); if (logWriter != null) { return logWriter; @@ -141,34 +148,32 @@ public class ArchiverHandler extends AbstractLogHandler * This method is just a fancy way of generating a stripped * down number representing year, month, day and hour in order * to partition logging in time. - * - * <P> + * <p> * This method is not thread-safe. */ - public int dateHash (long time) { + public int dateHash(long time) { calendar.setTimeInMillis(time); - int year = calendar.get(Calendar.YEAR); - int month = calendar.get(Calendar.MONTH) + 1; - int day = calendar.get(Calendar.DAY_OF_MONTH); - int hour = calendar.get(Calendar.HOUR_OF_DAY); + int year = calendar.get(Calendar.YEAR); + int month = calendar.get(Calendar.MONTH) + 1; + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); return year * 1000000 - + month * 10000 - + day * 100 - + hour; + + month * 10000 + + day * 100 + + hour; } /** * Generate prefix for log filenames based on log message. - * - * <P> + * <p> * <EM>This message is <code>public</code> only because we need * access to it in unit tests. For all practical purposes this * method does not exist to you, the application programmer, OK? :-)</EM> - * <P> + * <p> * XXX optimize! */ - public String getPrefix (LogMessage msg) { + public String getPrefix(LogMessage msg) { calendar.setTimeInMillis(msg.getTime()); /* int year = calendar.get(Calendar.YEAR); @@ -177,50 +182,47 @@ public class ArchiverHandler extends AbstractLogHandler int hour = calendar.get(Calendar.HOUR_OF_DAY); */ StringBuffer result = new StringBuffer(absoluteRootDir.length() - + 1 // slash - + 4 // year - + 1 // slash - + 2 // month - + 1 // slash - + 2 // day - + 1 // slash - + 2 // hour + + 1 // slash + + 4 // year + + 1 // slash + + 2 // month + + 1 // slash + + 2 // day + + 1 // slash + + 2 // hour ) .append(absoluteRootDir).append("/") .append(dateformat.format(calendar.getTime())); return result.toString(); } - public boolean doHandle (LogMessage msg) { + public boolean doHandle(LogMessage msg) { try { LogWriter logWriter = getLogWriter(msg); logWriter.write(msg.toString()); - } - catch (IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } return true; } - public synchronized void flush () { + public synchronized void flush() { for (LogWriter l : logWriterLRUCache.values()) { try { l.flush(); - } - catch (IOException e) { + } catch (IOException e) { log.log(Level.WARNING, "Flushing failed", e); } } } - public synchronized void close () { + public synchronized void close() { Iterator<LogWriter> it = logWriterLRUCache.values().iterator(); while (it.hasNext()) { LogWriter l = it.next(); try { l.close(); - } - catch (IOException e) { + } catch (IOException e) { log.log(Level.WARNING, "Closing failed", e); } it.remove(); @@ -249,7 +251,7 @@ public class ArchiverHandler extends AbstractLogHandler } - public String toString () { + public String toString() { return ArchiverHandler.class.getName() + ": root=" + absoluteRootDir; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java index 0d04b4cf981..3c3bd80e607 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java @@ -2,13 +2,13 @@ package com.yahoo.logserver.handlers.archive; import java.util.logging.Logger; + import com.yahoo.logserver.Server; import com.yahoo.plugin.Config; import com.yahoo.plugin.Plugin; -public class ArchiverPlugin implements Plugin -{ +public class ArchiverPlugin implements Plugin { /** * Default log archive dir (relative to current directory * at startup). @@ -21,8 +21,7 @@ public class ArchiverPlugin implements Plugin private static final String DEFAULT_MAXFILESIZE = "20971520"; private final Server server = Server.getInstance(); - private static final Logger log = - Logger.getLogger(ArchiverPlugin.class.getName()); + private static final Logger log = Logger.getLogger(ArchiverPlugin.class.getName()); private ArchiverHandler archiver; /** @@ -32,20 +31,21 @@ public class ArchiverPlugin implements Plugin return "logarchive"; } - /** Initialize the archiver plugin - * + /** + * Initialize the archiver plugin + * <p> * Config keys used: - * - * maxfilesize - * dir The root of the logarchive, make sure this does - * <b>not</b> end with a '/' character. + * <p> + * maxfilesize + * dir The root of the logarchive, make sure this does + * <b>not</b> end with a '/' character. */ public void initPlugin(Config config) { if (archiver != null) { - log.finer("ArchivePlugin doubly initialized"); + log.finer("ArchivePlugin doubly initialized"); throw new IllegalStateException("plugin already initialized: " - + getPluginName()); + + getPluginName()); } // Possible to disable logarchive for testing @@ -65,9 +65,9 @@ public class ArchiverPlugin implements Plugin public void shutdownPlugin() { if (archiver == null) { - log.finer("ArchiverPlugin shutdown before initialize"); + log.finer("ArchiverPlugin shutdown before initialize"); throw new IllegalStateException("plugin not initialized: " - + getPluginName()); + + getPluginName()); } server.unregisterLogHandler(archiver); server.unregisterFlusher(archiver); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index fb78e8243c3..17af537026a 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -12,23 +12,21 @@ import com.yahoo.log.LogLevel; /** * This class is not thread-safe. * - * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ -public class LogWriter extends Writer -{ +public class LogWriter extends Writer { private static final Logger log = Logger.getLogger(LogWriter.class.getName()); - private long bytesWritten = 0; - private int generation = 0; - private int maxSize = 20 * (1024*1024); - private final int resumeLimit = 95; - private final int resumeLimitSize = (maxSize * resumeLimit / 100); + private long bytesWritten = 0; + private int generation = 0; + private int maxSize = 20 * (1024 * 1024); + private final int resumeLimit = 95; + private final int resumeLimitSize = (maxSize * resumeLimit / 100); private File currentFile; private Writer writer; private final String prefix; - public LogWriter (String prefix, int maxSize) throws IOException { + public LogWriter(String prefix, int maxSize) throws IOException { this.prefix = prefix; this.maxSize = maxSize; writer = nextWriter(); @@ -38,14 +36,12 @@ public class LogWriter extends Writer * This is called when we want to rotate the output file to * start writing the next file. There are two scenarios when * we do this: - * * <UL> - * <LI> initial case, when we have no file - * <LI> when we have filled the file and want to rotate it + * <LI> initial case, when we have no file + * <LI> when we have filled the file and want to rotate it * </UL> - * */ - private Writer nextWriter () throws IOException { + private Writer nextWriter() throws IOException { if (writer != null) { writer.close(); @@ -64,7 +60,7 @@ public class LogWriter extends Writer // if compressed version exists we skip it if ((new File(name + ".gz").exists()) - || (new File(name + ".bz2").exists())) { + || (new File(name + ".bz2").exists())) { continue; } @@ -91,9 +87,9 @@ public class LogWriter extends Writer } else { log.fine("nextWriter, not resuming " + name - + " because it is bigger than " - + resumeLimit - + " percent of max"); + + " because it is bigger than " + + resumeLimit + + " percent of max"); } } @@ -104,13 +100,12 @@ public class LogWriter extends Writer * Note that this method should not be used directly since * that would circumvent rotation when it grows past its * maximum size. use the one that takes String instead. - * - * <P> + * <p> * <em> * (This is a class which is only used internally anyway) * </em> */ - public void write (char[] cbuff, int offset, int len) throws IOException { + public void write(char[] cbuff, int offset, int len) throws IOException { throw new RuntimeException("This method should not be used"); } @@ -124,14 +119,14 @@ public class LogWriter extends Writer if (bytesWritten >= maxSize) { log.fine("logfile '" - + currentFile.getAbsolutePath() - + "' full, rotating"); + + currentFile.getAbsolutePath() + + "' full, rotating"); writer = nextWriter(); } } - public void flush() throws IOException{ + public void flush() throws IOException { if (writer != null) { writer.flush(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java index d36486e0c44..0b2e3e8f1b5 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriterLRUCache.java @@ -8,23 +8,20 @@ import java.util.LinkedHashMap; import java.util.Map; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ @SuppressWarnings("serial") -public class LogWriterLRUCache extends LinkedHashMap<Integer, LogWriter> -{ - private static final Logger log - = Logger.getLogger(LogWriterLRUCache.class.getName()); +public class LogWriterLRUCache extends LinkedHashMap<Integer, LogWriter> { + private static final Logger log = Logger.getLogger(LogWriterLRUCache.class.getName()); final int maxEntries = 100; - public LogWriterLRUCache(int initialCapacity, - float loadFactor) { + public LogWriterLRUCache(int initialCapacity, float loadFactor) { super(initialCapacity, loadFactor, true); } // TODO: implement unit test for this - protected boolean removeEldestEntry (Map.Entry<Integer, LogWriter> eldest) { + protected boolean removeEldestEntry(Map.Entry<Integer, LogWriter> eldest) { if (size() > maxEntries) { LogWriter logWriter = eldest.getValue(); log.fine("Closing oldest LogWriter: " + logWriter); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index ff587bdc396..4869896cfa6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -5,9 +5,7 @@ package com.yahoo.logserver.handlers.logmetrics; import java.util.Arrays; -import java.util.Iterator; import java.util.List; -import java.util.Set; import java.util.Map; import java.util.TreeMap; import java.util.ArrayList; @@ -24,25 +22,22 @@ import com.yahoo.logserver.handlers.AbstractLogHandler; * The LogMetricsHandler stores a count of the number of log messages * per level per host and sends an event count for this five minutes. * - * * @author hmusum */ -public class LogMetricsHandler extends AbstractLogHandler -{ - public static final long EVENTINTERVAL = 5 * 60; // in seconds +public class LogMetricsHandler extends AbstractLogHandler { + private static final long EVENTINTERVAL = 5 * 60; // in seconds - private static final Logger log = - Logger.getLogger(LogMetricsHandler.class.getName()); + private static final Logger log = Logger.getLogger(LogMetricsHandler.class.getName()); // A list of log metrics per host and per log level private final List<LevelCount> logMetrics = new ArrayList<LevelCount>(); // The log levels that are handled by this plugin private static final Level[] levels = {LogLevel.INFO, - LogLevel.WARNING, - LogLevel.SEVERE, - LogLevel.ERROR, - LogLevel.FATAL}; + LogLevel.WARNING, + LogLevel.SEVERE, + LogLevel.ERROR, + LogLevel.FATAL}; /** @@ -67,7 +62,7 @@ public class LogMetricsHandler extends AbstractLogHandler String host = message.getHost(); Level logLevel = message.getLevel(); - boolean found = false; + boolean found = false; if (logMetrics.size() > 0) { LevelCount count; // Loop through the list logMetrics and check if there @@ -75,19 +70,19 @@ public class LogMetricsHandler extends AbstractLogHandler for (int i = 0; i < logMetrics.size(); i++) { count = logMetrics.get(i); if (count.getHost().equals(host) && - count.getLevel().getName().equals(logLevel.getName())) { + count.getLevel().getName().equals(logLevel.getName())) { count.addCount(1); found = true; break; } } } - + // There is no element in logMetrics with the same host and // level as in the message, so create a new object and add it // to the list. - if (!found) { - for (Level level : Arrays.asList(levels)) { + if (! found) { + for (Level level : Arrays.asList(levels)) { LevelCount levelCount; if (level.getName().equals(logLevel.getName())) { levelCount = new LevelCount(host, @@ -97,29 +92,27 @@ public class LogMetricsHandler extends AbstractLogHandler levelCount = new LevelCount(host, level, 0); - + } logMetrics.add(levelCount); } - } + } return true; } /** - * * Create event count for each log level and report it. For now we * add up the numbers for all host on each level and report that. - * */ private void sendEvents() { - Map<String,Long> levelCount = getMetricsPerLevel(); - for (Map.Entry<String,Long> entry : levelCount.entrySet()) { + Map<String, Long> levelCount = getMetricsPerLevel(); + for (Map.Entry<String, Long> entry : levelCount.entrySet()) { String key = entry.getKey(); Long count = entry.getValue(); Event.count("log_message." + key.toLowerCase(), count.longValue()); } } - + public void flush() {} public void close() {} @@ -137,7 +130,7 @@ public class LogMetricsHandler extends AbstractLogHandler public long getMetricsCount() { long count = 0; for (LevelCount levelCount : logMetrics) { - count = count + levelCount.getCount(); + count = count + levelCount.getCount(); } return count; } @@ -148,18 +141,18 @@ public class LogMetricsHandler extends AbstractLogHandler * * @return A Map of log level counts */ - public Map<String,Long> getMetricsPerLevel() { + public Map<String, Long> getMetricsPerLevel() { Map<String, Long> levelCounts = new TreeMap<String, Long>(); // Loop through all levels summing the count for all hosts. for (Level level : Arrays.asList(levels)) { - String levelName = level.getName(); - long count = 0; - for (LevelCount levelCount : logMetrics) { - if (levelName.equals(levelCount.getLevel().getName())) { - count += levelCount.getCount(); - } - } - levelCounts.put(levelName, count); + String levelName = level.getName(); + long count = 0; + for (LevelCount levelCount : logMetrics) { + if (levelName.equals(levelCount.getLevel().getName())) { + count += levelCount.getCount(); + } + } + levelCounts.put(levelName, count); } return levelCounts; } @@ -167,21 +160,20 @@ public class LogMetricsHandler extends AbstractLogHandler /** * The LevelCount class represents the number (count) of log * messages with the same log level for a host. - * */ private class LevelCount { private final String host; private final Level level; private long count; - + LevelCount(String host, Level level, long count) { - this.host = host; - this.level = level; - this.count = count; + this.host = host; + this.level = level; + this.count = count; } - + LevelCount(String host, Level level) { - this(host, level, 0); + this(host, level, 0); } Level getLevel() { @@ -207,7 +199,7 @@ public class LogMetricsHandler extends AbstractLogHandler public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Host=" + host + ", level = " + level.getName() + - ",count=" + count); + ",count=" + count); return sb.toString(); } } @@ -215,7 +207,6 @@ public class LogMetricsHandler extends AbstractLogHandler /** * Implements a thread that sends events every EVENTINTERVAL * seconds. - * */ private class EventGenerator implements Runnable { public void run() { @@ -228,6 +219,6 @@ public class LogMetricsHandler extends AbstractLogHandler } sendEvents(); } - } + } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java index 204e35d52bb..51fac499903 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsPlugin.java @@ -11,10 +11,8 @@ import com.yahoo.plugin.Config; import com.yahoo.plugin.Plugin; -public class LogMetricsPlugin implements Plugin -{ - private static final Logger log = - Logger.getLogger(LogMetricsPlugin.class.getName()); +public class LogMetricsPlugin implements Plugin { + private static final Logger log = Logger.getLogger(LogMetricsPlugin.class.getName()); private LogMetricsHandler logMetricsHandler; private final Server server = Server.getInstance(); @@ -22,15 +20,15 @@ public class LogMetricsPlugin implements Plugin return "logmetrics"; } - /** Initialize the logmetrics plugin + /** + * Initialize the logmetrics plugin * * @param config Plugin config object, keys used: - * <code>thread</code> - name of handler thread this plugin runs in - * + * <code>thread</code> - name of handler thread this plugin runs in */ public void initPlugin(Config config) { if (logMetricsHandler != null) { - log.finer("LogMetricsPlugin doubly initialized"); + log.finer("LogMetricsPlugin doubly initialized"); throw new IllegalStateException( "plugin already initialized: " + getPluginName()); } @@ -44,9 +42,9 @@ public class LogMetricsPlugin implements Plugin */ public void shutdownPlugin() { if (logMetricsHandler == null) { - log.finer("LogMetricsPlugin shutdown before initialize"); + log.finer("LogMetricsPlugin shutdown before initialize"); throw new IllegalStateException("plugin not initialized: " + - getPluginName()); + getPluginName()); } server.unregisterLogHandler(logMetricsHandler); logMetricsHandler = null; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java index f20adbe3c91..daf408c17a4 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCache.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.IdentityHashMap; import java.nio.ByteBuffer; import java.nio.charset.Charset; + import com.yahoo.log.LogMessage; import com.yahoo.logserver.formatter.LogFormatter; import com.yahoo.logserver.formatter.LogFormatterManager; @@ -15,16 +16,14 @@ import com.yahoo.logserver.formatter.LogFormatterManager; * is to make it easier to support multiple message formats while * still ensuring we don't format more messages than we strictly need * to and that we don't keep around more buffers that we ought to. - * - * <P> + * <p> * This is not a general purpose class, I think, so please * refer to the source code of the Replicator class for * information on how to use this. - * - * <P> + * <p> * This class is not threadsafe. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class FormattedBufferCache { // the documentation says " All of the methods defined in this @@ -35,7 +34,7 @@ public class FormattedBufferCache { private final IdentityHashMap<LogFormatter, ByteBuffer> buffers; - public FormattedBufferCache () { + public FormattedBufferCache() { // hope this is a good hash size int initialSize = LogFormatterManager.getFormatterNames().length * 2; buffers = new IdentityHashMap<LogFormatter, ByteBuffer>(initialSize); @@ -47,11 +46,11 @@ public class FormattedBufferCache { * exist in the cache from before, it will after this * method returns. * - * @param msg The log message you wish to format + * @param msg The log message you wish to format * @param formatter The log formatter you wish to use for formatting * @return Returns a ByteBuffer slice */ - public ByteBuffer getFormatted (LogMessage msg, LogFormatter formatter) { + public ByteBuffer getFormatted(LogMessage msg, LogFormatter formatter) { ByteBuffer bb = buffers.get(formatter); if (bb == null) { bb = charset.encode(formatter.format(msg)); @@ -65,14 +64,14 @@ public class FormattedBufferCache { * clients we clear the cache so we are ready for the next * message. */ - public void reset () { + public void reset() { buffers.clear(); } /** * This is here for test purposes. Don't get any bright ideas. */ - public Map<LogFormatter,ByteBuffer> getUnderlyingMapOnlyForTesting() { + public Map<LogFormatter, ByteBuffer> getUnderlyingMapOnlyForTesting() { return buffers; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java index f8dc700931f..1d8da93c483 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/Replicator.java @@ -6,6 +6,7 @@ import java.nio.channels.SocketChannel; import java.util.HashSet; import java.util.Set; import java.util.logging.Logger; + import com.yahoo.io.Connection; import com.yahoo.io.ConnectionFactory; import com.yahoo.io.Listener; @@ -16,115 +17,109 @@ import com.yahoo.logserver.handlers.AbstractLogHandler; /** * The Replicator plugin is used for replicating log messages sent * to the logserver. - * - * <P> + * <p> * Per default the replicator will start dropping messages enqueued * to a client if the outbound message queue reaches 5000 messages. * This limit can be configured by setting the system property * <code>logserver.replicator.maxqueuelength</code> to the desired * value. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ -public class Replicator extends AbstractLogHandler -implements ConnectionFactory -{ - private static final Logger log = Logger.getLogger(Replicator.class.getName()); +public class Replicator extends AbstractLogHandler implements ConnectionFactory { + private static final Logger log = Logger.getLogger(Replicator.class.getName()); - private int port; - private Listener listener; - private final Set<ReplicatorConnection> connections = new HashSet<ReplicatorConnection>(); - private final FormattedBufferCache bufferCache = new FormattedBufferCache(); + private int port; + private Listener listener; + private final Set<ReplicatorConnection> connections = new HashSet<ReplicatorConnection>(); + private final FormattedBufferCache bufferCache = new FormattedBufferCache(); - /** - * @param port The port to which the replicator listens. - * - */ - public Replicator (int port) throws IOException { - this.port = port; - listen(port); - } + /** + * @param port The port to which the replicator listens. + */ + public Replicator(int port) throws IOException { + this.port = port; + listen(port); + } - public Replicator () { - } + public Replicator() { + } - public void listen (int port) throws IOException { - if (listener != null) { - throw new IllegalStateException("already listening to port " + this.port); - } - listener = new Listener("replicator"); - listener.listen(this, port); - listener.start(); - log.log(LogLevel.CONFIG, "port=" + port); - } + public void listen(int port) throws IOException { + if (listener != null) { + throw new IllegalStateException("already listening to port " + this.port); + } + listener = new Listener("replicator"); + listener.listen(this, port); + listener.start(); + log.log(LogLevel.CONFIG, "port=" + port); + } - public synchronized boolean doHandle (LogMessage msg) { - boolean logged = false; - bufferCache.reset(); - for (ReplicatorConnection c : connections) { - try { - if (c.isLoggable(msg)) { - c.enqueue(bufferCache.getFormatted(msg, c.formatter)); - logged = true; - } - } - catch (IOException e) { - log.log(LogLevel.DEBUG, "Writing failed", e); - } - } - return logged; - } + public synchronized boolean doHandle(LogMessage msg) { + boolean logged = false; + bufferCache.reset(); + for (ReplicatorConnection c : connections) { + try { + if (c.isLoggable(msg)) { + c.enqueue(bufferCache.getFormatted(msg, c.formatter)); + logged = true; + } + } catch (IOException e) { + log.log(LogLevel.DEBUG, "Writing failed", e); + } + } + return logged; + } - public void close() { - // kill the listener thread, then wait for it to - // shut down. - try { - listener.interrupt(); - listener.join(); - log.log(LogLevel.DEBUG, "Replicator listener stopped"); - } - catch (InterruptedException e) { - log.log(LogLevel.WARNING, - "Replicator listener was interrupted", - e); - } - } + public void close() { + // kill the listener thread, then wait for it to + // shut down. + try { + listener.interrupt(); + listener.join(); + log.log(LogLevel.DEBUG, "Replicator listener stopped"); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, + "Replicator listener was interrupted", + e); + } + } - /** - * Currently a NOP, but we might want to have some best-effort - * mechanism for trying to flush all connections within some - * time-frame. - */ - public void flush() {} + /** + * Currently a NOP, but we might want to have some best-effort + * mechanism for trying to flush all connections within some + * time-frame. + */ + public void flush() {} - /** - * Factory method for wrapping new connections in the proper - * (Replicator)Connection objects. - * - * @param socket The new SocketChannel - * @param listener The Listener instance we want to use - */ - public synchronized Connection newConnection (SocketChannel socket, - Listener listener) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.fine("New replicator connection: " + socket); - } - ReplicatorConnection n = - new ReplicatorConnection(socket, listener, this); - connections.add(n); - return n; - } + /** + * Factory method for wrapping new connections in the proper + * (Replicator)Connection objects. + * + * @param socket The new SocketChannel + * @param listener The Listener instance we want to use + */ + public synchronized Connection newConnection(SocketChannel socket, + Listener listener) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.fine("New replicator connection: " + socket); + } + ReplicatorConnection n = + new ReplicatorConnection(socket, listener, this); + connections.add(n); + return n; + } - /** - * Removes a ReplicatorConnection from the set of active - * connections. - */ - protected synchronized void deRegisterConnection (ReplicatorConnection conn) { - connections.remove(conn); - } + /** + * Removes a ReplicatorConnection from the set of active + * connections. + */ + protected synchronized void deRegisterConnection(ReplicatorConnection conn) { + connections.remove(conn); + } - public String toString () { - return Replicator.class.getName(); - } + public String toString() { + return Replicator.class.getName(); + } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java index 2978f751665..003d35e96d0 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorConnection.java @@ -23,18 +23,22 @@ import com.yahoo.logserver.formatter.LogFormatterManager; /** * Replication client connection. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ -public class ReplicatorConnection implements Connection, LogFilter -{ - private static final Logger log - = Logger.getLogger(ReplicatorConnection.class.getName()); +public class ReplicatorConnection implements Connection, LogFilter { + private static final Logger log = Logger.getLogger(ReplicatorConnection.class.getName()); - /** The maximum number of queued messages before we start dropping */ + /** + * The maximum number of queued messages before we start dropping + */ private static final int maxQueueLength; - /** The maximum number of times we go over maxQueueLength before we log a warning */ + /** + * The maximum number of times we go over maxQueueLength before we log a warning + */ private static final int maxRetriesBeforeWarning = 10; - /** Count of how many times we have received a message while the queue is full */ + /** + * Count of how many times we have received a message while the queue is full + */ private static int queueFullCount = 0; static { @@ -64,11 +68,8 @@ public class ReplicatorConnection implements Connection, LogFilter * Constructs a ReplicatorConnection. Note that initially the * filter of this connection is set to MuteFilter, which mutes * all log messages. - * */ - public ReplicatorConnection (SocketChannel socket, - Listener listener, - Replicator replicator) { + public ReplicatorConnection(SocketChannel socket, Listener listener, Replicator replicator) { this.socket = socket; this.listener = listener; this.replicator = replicator; @@ -97,9 +98,8 @@ public class ReplicatorConnection implements Connection, LogFilter * serialize it into. * * @param msg The log message offered - * */ - public boolean isLoggable (LogMessage msg) { + public boolean isLoggable(LogMessage msg) { if (filter == null) { return true; } @@ -109,7 +109,7 @@ public class ReplicatorConnection implements Connection, LogFilter /** * Return the description of the currently active filter. */ - public String description () { + public String description() { if (filter == null) { return "No filter defined"; } @@ -124,7 +124,7 @@ public class ReplicatorConnection implements Connection, LogFilter * @param buffer the ByteBuffer into which the log message is * serialized. */ - public synchronized void enqueue (ByteBuffer buffer) throws IOException { + public synchronized void enqueue(ByteBuffer buffer) throws IOException { if (writeBuffer == null) { writeBuffer = buffer; } else { @@ -133,7 +133,7 @@ public class ReplicatorConnection implements Connection, LogFilter queueFullCount++; if (! droppingMode) { droppingMode = true; - String message = "client at " + remoteHost + " can't keep up, dropping messages"; + String message = "client at " + remoteHost + " can't keep up, dropping messages"; if (queueFullCount > maxRetriesBeforeWarning) { log.log(LogLevel.WARNING, message); queueFullCount = 0; @@ -154,38 +154,38 @@ public class ReplicatorConnection implements Connection, LogFilter } - public void read () throws IOException { + public void read() throws IOException { if (! readBuffer.hasRemaining()) { log.warning("Log message too long. Message exceeds " - + readBuffer.capacity() - + " bytes. Connection dropped."); + + readBuffer.capacity() + + " bytes. Connection dropped."); close(); return; } - int ret = socket.read(readBuffer); - if (ret == -1) { - close(); - return; - } + int ret = socket.read(readBuffer); + if (ret == - 1) { + close(); + return; + } - if (ret == 0) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.fine("zero byte read occurred"); - } - } + if (ret == 0) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.fine("zero byte read occurred"); + } + } - readBuffer.flip(); + readBuffer.flip(); - String s; - while ((s = ReadLine.readLine(readBuffer)) != null) { - onCommand(s); - } + String s; + while ((s = ReadLine.readLine(readBuffer)) != null) { + onCommand(s); + } } - public synchronized void write () throws IOException { + public synchronized void write() throws IOException { if (! socket.isOpen()) { // throw new IllegalStateException("SocketChannel not open in write()"); close(); @@ -218,8 +218,7 @@ public class ReplicatorConnection implements Connection, LogFilter // try { bytesWritten = socket.write(writeBuffer); - } - catch (IOException e) { + } catch (IOException e) { close(); return; } @@ -227,13 +226,13 @@ public class ReplicatorConnection implements Connection, LogFilter // buffer drained so we forget it and see what happens when we // go around. if indeed we go around - if ((writeBuffer != null) && (!writeBuffer.hasRemaining())) { + if ((writeBuffer != null) && (! writeBuffer.hasRemaining())) { writeBuffer = null; } } while (bytesWritten > 0); } - public synchronized void close () throws IOException { + public synchronized void close() throws IOException { replicator.deRegisterConnection(this); socket.close(); writeBuffer = null; @@ -241,15 +240,15 @@ public class ReplicatorConnection implements Connection, LogFilter log.log(LogLevel.DEBUG, "closing connection to " + remoteHost); } - public int selectOps () { + public int selectOps() { return SelectionKey.OP_READ; } - public SocketChannel socketChannel () { + public SocketChannel socketChannel() { return socket; } - public void connect () { + public void connect() { } @@ -257,7 +256,7 @@ public class ReplicatorConnection implements Connection, LogFilter // ==== command processing // ======================================================== - void onCommand (String s) { + void onCommand(String s) { log.log(LogLevel.DEBUG, "COMMAND: '" + s + "' from " + remoteHost); StringTokenizer st = new StringTokenizer(s.toLowerCase()); while (st.hasMoreTokens()) { @@ -308,7 +307,7 @@ public class ReplicatorConnection implements Connection, LogFilter } } - void onFormatter (String formatterName) { + void onFormatter(String formatterName) { LogFormatter newFormatter = LogFormatterManager.getLogFormatter(formatterName); if (newFormatter == null) { print("# 405 formatter not found '" + formatterName + "'\n"); @@ -319,7 +318,7 @@ public class ReplicatorConnection implements Connection, LogFilter print("# 202 using '" + formatter + "'\n"); } - void onUse (String filterName) { + void onUse(String filterName) { LogFilter newFilter = LogFilterManager.getLogFilter(filterName); if (newFilter == null) { print("# 404 filter not found '" + filterName + "'\n"); @@ -331,8 +330,7 @@ public class ReplicatorConnection implements Connection, LogFilter } - - void onList () { + void onList() { print("# 203 filter list\n"); String filterNames[] = LogFilterManager.getFilterNames(); for (int i = 0; i < filterNames.length; i++) { @@ -342,7 +340,7 @@ public class ReplicatorConnection implements Connection, LogFilter print("# 205 end filter list\n"); } - void onListFormatters () { + void onListFormatters() { print("# 206 formatter list\n"); String formatterNames[] = LogFormatterManager.getFormatterNames(); for (int i = 0; i < formatterNames.length; i++) { @@ -352,60 +350,58 @@ public class ReplicatorConnection implements Connection, LogFilter print("# 208 end formatter list\n"); } - private void print (String s) { + private void print(String s) { try { enqueue(IOUtils.utf8ByteBuffer(s)); - } - catch (IOException e) { + } catch (IOException e) { log.log(LogLevel.WARNING, "error printing", e); try { close(); - } - catch (IOException e2) { + } catch (IOException e2) { // ignore } } } - void onStats () { + void onStats() { print(new StringBuilder(80) - .append("# 206 stats start (this connection)\n") - .append("# 207 ").append(numHandled).append(" handled\n") - .append("# 208 ").append(numDropped).append(" dropped\n") - .append("# 209 ").append(numQueued) - .append(" handled and queued\n") - .append("# 210 ").append(totalBytesWritten) - .append(" total bytes written\n") - .append("# 211 stats end\n") - .toString() - ); + .append("# 206 stats start (this connection)\n") + .append("# 207 ").append(numHandled).append(" handled\n") + .append("# 208 ").append(numDropped).append(" dropped\n") + .append("# 209 ").append(numQueued) + .append(" handled and queued\n") + .append("# 210 ").append(totalBytesWritten) + .append(" total bytes written\n") + .append("# 211 stats end\n") + .toString() + ); } - public int getNumHandled () { + public int getNumHandled() { return numHandled; } - public int getNumQueued () { + public int getNumQueued() { return numQueued; } - public int getNumDropped () { + public int getNumDropped() { return numDropped; } - public long getTotalBytesWritten () { + public long getTotalBytesWritten() { return totalBytesWritten; } - public String getLogFilterName () { + public String getLogFilterName() { return filterName; } - + void setFilter(LogFilter filter) { this.filter = filter; } - + } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java index 03bc66c8262..b1fe0ce9193 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/replicator/ReplicatorPlugin.java @@ -9,23 +9,21 @@ import com.yahoo.logserver.Server; import com.yahoo.plugin.Config; import com.yahoo.plugin.Plugin; -public class ReplicatorPlugin implements Plugin -{ +public class ReplicatorPlugin implements Plugin { private static final String DEFAULT_PORT = "19083"; - private static final Logger log = - Logger.getLogger(ReplicatorPlugin.class.getName()); + private static final Logger log = Logger.getLogger(ReplicatorPlugin.class.getName()); + private Replicator replicator; private final Server server = Server.getInstance(); - public String getPluginName () { + public String getPluginName() { return "replicator"; } /** * Initialize the replicator plugin */ - public void initPlugin (Config config) { - + public void initPlugin(Config config) { if (replicator != null) { throw new IllegalStateException( "plugin already initialized: " + getPluginName()); @@ -34,8 +32,7 @@ public class ReplicatorPlugin implements Plugin String threadName = config.get("thread", getPluginName()); try { replicator = new Replicator(listenPort); - } - catch (IOException e) { + } catch (IOException e) { log.log(LogLevel.WARNING, "init failed: " + e); return; } @@ -45,7 +42,7 @@ public class ReplicatorPlugin implements Plugin /** * Shut down the replicator plugin. */ - public void shutdownPlugin () { + public void shutdownPlugin() { if (replicator == null) { throw new IllegalStateException( diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java index f9853ea03b0..be59fe5222f 100644 --- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java +++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnection.java @@ -31,7 +31,7 @@ import java.nio.channels.SelectionKey; * analyzed and errors can be corrected. * </UL> * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogConnection implements Connection { diff --git a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java index 5d97420f30a..7f0f18a0435 100644 --- a/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java +++ b/logserver/src/main/java/com/yahoo/logserver/net/LogConnectionFactory.java @@ -13,29 +13,26 @@ import java.util.logging.Logger; import java.nio.channels.SocketChannel; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ +public class LogConnectionFactory implements ConnectionFactory { + private static final Logger log = Logger.getLogger(LogConnectionFactory.class.getName()); -public class LogConnectionFactory implements ConnectionFactory -{ - private static final Logger log - = Logger.getLogger(LogConnectionFactory.class.getName()); + private final LogDispatcher dispatcher; + private final Levels defaultLogLevels; - final LogDispatcher dispatcher; - final Levels defaultLogLevels; - - public LogConnectionFactory (LogDispatcher dispatcher) { + public LogConnectionFactory(LogDispatcher dispatcher) { this.dispatcher = dispatcher; defaultLogLevels = Levels.parse(System.getProperty("logserver.default.loglevels", "")); } - public Connection newConnection (SocketChannel socket, Listener listener) { + public Connection newConnection(SocketChannel socket, Listener listener) { if (log.isLoggable(Level.FINE)) { log.fine("New connection: " + socket); } return new LogConnection(socket, listener, dispatcher, - (Levels)defaultLogLevels.clone()); + (Levels) defaultLogLevels.clone()); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java index c9b749f4417..985f7159fc4 100644 --- a/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java +++ b/logserver/src/main/java/com/yahoo/logserver/net/control/Levels.java @@ -8,7 +8,7 @@ import java.util.Map; * This class is used to represent the state of each log level * in a set of states. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Levels implements Cloneable { private final Map<String, State> levelsMap = new LinkedHashMap<String, State>(10); diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java index 623bf04aa36..d8b05555133 100644 --- a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -15,7 +15,7 @@ import com.yahoo.log.LogMessage; * in a log file are correct. Any incorrectly formatted log * message is output to stdout. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class VerifyLogfile { diff --git a/logserver/src/main/java/com/yahoo/plugin/Config.java b/logserver/src/main/java/com/yahoo/plugin/Config.java index a7ea2b44caf..9cae0606809 100644 --- a/logserver/src/main/java/com/yahoo/plugin/Config.java +++ b/logserver/src/main/java/com/yahoo/plugin/Config.java @@ -5,10 +5,9 @@ package com.yahoo.plugin; * This interface specifies an API for configuring runtime-loadable * server plugins. * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ -public abstract class Config -{ +public abstract class Config { /** * @return a config value for the specified key */ diff --git a/logserver/src/main/java/com/yahoo/plugin/Plugin.java b/logserver/src/main/java/com/yahoo/plugin/Plugin.java index e53bc3a1faa..ced89b7fe1a 100644 --- a/logserver/src/main/java/com/yahoo/plugin/Plugin.java +++ b/logserver/src/main/java/com/yahoo/plugin/Plugin.java @@ -8,26 +8,25 @@ package com.yahoo.plugin; * calls application-specific registration methods to connect the * plugin to the hosting application. * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ -public interface Plugin -{ +public interface Plugin { /** * @return a unique and simple name for the plugin */ - public String getPluginName(); + String getPluginName(); /** * Initialize the plugin. * * @param config Config object for this plugin */ - public void initPlugin(Config config); + void initPlugin(Config config); /** * Shut down the plugin. Must clean up all resources allocated by * initPlugin() or any of the handler methods. */ - public void shutdownPlugin(); + void shutdownPlugin(); } diff --git a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java index 763dc2d1f3e..4f17678e3b9 100644 --- a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java +++ b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java @@ -8,15 +8,14 @@ package com.yahoo.plugin; * then return the contents of the "logserver.archiver.foo" system * property. * - * @author <a href="mailto:stig@yahoo-inc.com">Stig Bakken</a> + * @author Stig Bakken */ -public class SystemPropertyConfig extends Config -{ +public class SystemPropertyConfig extends Config { private final String prefix; /** * @param prefix Prefix string prepended to config keys - * as they are looked up as system properties. + * as they are looked up as system properties. */ public SystemPropertyConfig(String prefix) { this.prefix = prefix; @@ -29,7 +28,7 @@ public class SystemPropertyConfig extends Config return System.getProperty(prefix + key, defaultValue); } - public String toString () { + public String toString() { return "Prefix=" + prefix; } } diff --git a/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java index 79d16de61f0..d6953fd3bc3 100644 --- a/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/FlusherTestCase.java @@ -19,7 +19,7 @@ public class FlusherTestCase { Thread.sleep(5000); flusher.interrupt(); flusher.join(); - assertTrue(handler.flushCalled>0); + assertTrue(handler.flushCalled > 0); } } diff --git a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java index 1a6cd304d79..4d0c08ac151 100644 --- a/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/ServerTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.logserver; import com.yahoo.log.LogSetup; -import com.yahoo.logserver.Server; import com.yahoo.logserver.handlers.LogHandler; import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; import com.yahoo.logserver.test.LogDispatcherTestCase; @@ -10,17 +9,18 @@ import com.yahoo.logserver.test.LogDispatcherTestCase; import java.io.IOException; import org.junit.*; + import static org.junit.Assert.*; /** * Unit tests for the Server class. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class ServerTestCase { @Test - public void testStartupAndRegHandlers() throws IOException, InterruptedException { + public void testStartupAndRegHandlers() throws IOException, InterruptedException { Server.help(); Server server = Server.getInstance(); server.initialize(18322); @@ -29,20 +29,19 @@ public class ServerTestCase { serverThread.start(); assertTrue(serverThread.isAlive()); LogHandler handler = new LogDispatcherTestCase.MockHandler(); - server.registerLogHandler(handler , "foo"); + server.registerLogHandler(handler, "foo"); assertEquals(Server.threadNameForHandler().get(handler), "foo"); server.unregisterLogHandler(handler); assertEquals(Server.threadNameForHandler().get(handler), null); - serverThread.interrupt(); - try { - serverThread.join(); - assertTrue(true); - } - catch (InterruptedException e) { - fail(); - } - } - + serverThread.interrupt(); + try { + serverThread.join(); + assertTrue(true); + } catch (InterruptedException e) { + fail(); + } + } + @Test public void testPluginLoaderClassLoading() { AbstractPluginLoader loader = new BuiltinPluginLoader(); @@ -51,5 +50,5 @@ public class ServerTestCase { System.setProperty("logserver.logmetrics.enable", "true"); loader.loadFromClass(LogMetricsPlugin.class); // Hm, no way to verify it was loaded } - + } diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java index 439c8e3978f..b2d4d322456 100644 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/LogFilterManagerTestCase.java @@ -10,12 +10,13 @@ import com.yahoo.logserver.filter.NullFilter; import com.yahoo.logserver.filter.MuteFilter; import org.junit.*; + import static org.junit.Assert.*; public class LogFilterManagerTestCase { @Test - public void testSystemFilters () { + public void testSystemFilters() { LogFilter f; f = LogFilterManager.getLogFilter("system.allevents"); diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java index fa22daac0d8..67733c8260b 100644 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/MetricsFilterTestCase.java @@ -12,25 +12,23 @@ import com.yahoo.log.LogMessage; import com.yahoo.logserver.filter.MetricsFilter; import org.junit.*; + import static org.junit.Assert.*; public class MetricsFilterTestCase { @Test - public void testValueEvents () - throws InvalidLogFormatException, IOException { + public void testValueEvents() throws InvalidLogFormatException, IOException { MetricsFilter filter = new MetricsFilter(); String filename = "src/test/files/value-events.txt"; BufferedReader br = new BufferedReader(new FileReader(filename)); - for (String line = br.readLine(); line != null; line = br.readLine()) - { + for (String line = br.readLine(); line != null; line = br.readLine()) { LogMessage m = LogMessage.parseNativeFormat(line); assertNotNull(m); try { Event event = m.getEvent(); assertNotNull(event); - } - catch (MalformedEventException e) { + } catch (MalformedEventException e) { fail(); } diff --git a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java index 5d6be5c16c7..ddd427ffb76 100644 --- a/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/filter/test/NoMetricsFilterTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.logserver.filter.MetricsFilter; import com.yahoo.logserver.filter.NoMetricsFilter; import org.junit.*; + import static org.junit.Assert.*; /** @@ -21,7 +22,7 @@ import static org.junit.Assert.*; public class NoMetricsFilterTestCase { @Test - public void testValueEvents () throws InvalidLogFormatException, IOException { + public void testValueEvents() throws InvalidLogFormatException, IOException { NoMetricsFilter filter = new NoMetricsFilter(); MetricsFilter metricsFilter = new MetricsFilter(); @@ -34,8 +35,7 @@ public class NoMetricsFilterTestCase { try { Event event = m.getEvent(); assertNotNull(event); - } - catch (MalformedEventException e) { + } catch (MalformedEventException e) { fail(); } diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java index 288300f9251..774cd68e995 100644 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/LogFormatterManagerTestCase.java @@ -11,12 +11,13 @@ import com.yahoo.logserver.formatter.NullFormatter; import com.yahoo.logserver.formatter.TextFormatter; import org.junit.*; + import static org.junit.Assert.*; /** * Test the LogFormatterManager * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogFormatterManagerTestCase { diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java index b0fc7ea09c0..3a02b7a1d05 100644 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/NullFormatterTestCase.java @@ -10,17 +10,18 @@ import com.yahoo.logserver.formatter.NullFormatter; import com.yahoo.logserver.test.MockLogEntries; import org.junit.*; + import static org.junit.Assert.*; /** * Test the NullFormatter * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class NullFormatterTestCase { @Test - public void testNullFormatter () { + public void testNullFormatter() { NullFormatter nf = new NullFormatter(); LogMessage[] ms = MockLogEntries.getMessages(); for (LogMessage m : ms) { diff --git a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java index 2e7e2d38b57..2e5f3ac3d8f 100644 --- a/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/formatter/test/TextFormatterTestCase.java @@ -11,12 +11,13 @@ import com.yahoo.logserver.formatter.TextFormatter; import com.yahoo.logserver.test.MockLogEntries; import org.junit.*; + import static org.junit.Assert.*; /** * Test the TextFormatter * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class TextFormatterTestCase { @@ -24,7 +25,7 @@ public class TextFormatterTestCase { * Just simple test to make sure it doesn't die on us */ @Test - public void testTextFormatter () { + public void testTextFormatter() { TextFormatter tf = new TextFormatter(); LogMessage[] ms = MockLogEntries.getMessages(); for (int i = 0; i < ms.length; i++) { @@ -36,7 +37,7 @@ public class TextFormatterTestCase { * Test that a specific log message is formatted correctly */ @Test - public void testSpecificMessage () throws InvalidLogFormatException { + public void testSpecificMessage() throws InvalidLogFormatException { String l = "1115200798.195568\texample.yahoo.com\t65819\ttopleveldispatch\tfdispatch.queryperf\tevent\tvalue/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825"; String result = "2005-05-04 09:59:58 example.yahoo.com 65819 topleveldispatch fdispatch.queryperf EVENT value/1 name=\"query_eval_time_avg_s\" value=0.0229635972697721825\n"; LogMessage m = LogMessage.parseNativeFormat(l); diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java index 930b6e7c5ad..e7bfb112848 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/HandlerThreadTestCase.java @@ -92,5 +92,5 @@ public class HandlerThreadTestCase { thread.interrupt(); thread.join(); } - + } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index ec1b0d962ef..0e29d9e80d4 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -21,15 +21,15 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.*; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class ArchiverHandlerTestCase { private static final String[] mStrings = { - "1095159244.095\thost\t1/2\tservice\tcomponent\tinfo\tpayload1", - "1095206399.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload2", - "1095206400.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload3", - "1095206401.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload4", + "1095159244.095\thost\t1/2\tservice\tcomponent\tinfo\tpayload1", + "1095206399.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload2", + "1095206400.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload3", + "1095206401.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload4", }; private static final LogMessage[] msg = new LogMessage[mStrings.length]; @@ -59,20 +59,19 @@ public class ArchiverHandlerTestCase { * [00:00:00.000, 23:59:59.999]. */ @Test - public void testDateHash () throws IOException { + public void testDateHash() throws IOException { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024); + 1024); long now = 1095159244095L; long midnight = 1095206400000L; assertEquals(2004091410, a.dateHash(now)); - assertEquals(2004091423, a.dateHash(midnight-1)); + assertEquals(2004091423, a.dateHash(midnight - 1)); assertEquals(2004091500, a.dateHash(midnight)); - assertEquals(2004091500, a.dateHash(midnight+1)); + assertEquals(2004091500, a.dateHash(midnight + 1)); a.close(); - } - finally { + } finally { IOUtils.recursiveDeleteDir(tmpDir); } } @@ -85,7 +84,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024); + 1024); LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\tthread\tservice\tcomponent\tinfo\tpayload"); LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\tthread\tservice\tcomponent\tinfo\tpayload"); assertEquals(tmpDir.getAbsolutePath() + "/2006/02/07/14", a.getPrefix(msg1)); @@ -94,8 +93,7 @@ public class ArchiverHandlerTestCase { a.close(); } catch (InvalidLogFormatException e) { fail(e.toString()); - } - finally { + } finally { IOUtils.recursiveDeleteDir(tmpDir); } } @@ -105,7 +103,7 @@ public class ArchiverHandlerTestCase { * written. */ @Test - public void testLogging () throws java.io.IOException, InvalidLogFormatException { + public void testLogging() throws java.io.IOException, InvalidLogFormatException { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), @@ -141,8 +139,7 @@ public class ArchiverHandlerTestCase { BufferedReader br = new BufferedReader(new FileReader(f)); for (String line = br.readLine(); line != null; - line = br.readLine()) - { + line = br.readLine()) { // primitive check if the messages match boolean foundMatch = false; for (int k = 0; k < mStrings.length; k++) { @@ -156,7 +153,7 @@ public class ArchiverHandlerTestCase { // try to instantiate messages to ensure that they // are parseable @SuppressWarnings("unused") - LogMessage m = LogMessage.parseNativeFormat(line); + LogMessage m = LogMessage.parseNativeFormat(line); messageCount++; } br.close(); @@ -165,8 +162,7 @@ public class ArchiverHandlerTestCase { // verify that the number of log messages written equals // the number of log messages we have in our test assertEquals(mStrings.length, messageCount); - } - finally { + } finally { IOUtils.recursiveDeleteDir(tmpDir); } } @@ -175,7 +171,7 @@ public class ArchiverHandlerTestCase { * Make sure that the file is rotated after N bytes */ @Test - public void testRotation () throws IOException { + public void testRotation() throws IOException { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), @@ -197,8 +193,7 @@ public class ArchiverHandlerTestCase { BufferedReader br = new BufferedReader(new FileReader(f)); for (String line = br.readLine(); line != null; - line = br.readLine()) - { + line = br.readLine()) { assertTrue(msg[1].toString().equals((line + "\n"))); msgCount++; } @@ -208,21 +203,20 @@ public class ArchiverHandlerTestCase { // make sure we have no more than 3 files assertTrue(! (new File(prefix + "-3").exists())); - } - finally { + } finally { IOUtils.recursiveDeleteDir(tmpDir); } } - + @Test public void testCacheEldestEntry() throws IOException { LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75); - for (int i = 0 ; i < cache.maxEntries+10 ; i++) { + for (int i = 0; i < cache.maxEntries + 10; i++) { cache.put(i, new LogWriter("/tmp/", 5)); } assertEquals(cache.size(), cache.maxEntries); } - + @Test public void testArchiverPlugin() { ArchiverPlugin ap = new ArchiverPlugin(); @@ -239,5 +233,5 @@ public class ArchiverHandlerTestCase { } ap.shutdownPlugin(); } - + } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/test/.gitignore b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/test/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/test/.gitignore +++ /dev/null diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java index a0e97c28f76..507c087b469 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java @@ -26,9 +26,9 @@ public class LastErrorsHolderTestCase { private static final int serverPort = 18324; private static final int lastErrorsHolderPort = 18326; - Server server; - Thread serverThread; - LastErrorsHolder lastErrorsHolder; + private Server server; + private Thread serverThread; + private LastErrorsHolder lastErrorsHolder; @Before public void setUp() throws InterruptedException, IOException { @@ -41,11 +41,11 @@ public class LastErrorsHolderTestCase { @After public void tearDown() throws InterruptedException { - if (serverThread!=null) { + if (serverThread != null) { serverThread.interrupt(); serverThread.join(); } - if (lastErrorsHolder !=null) lastErrorsHolder.close(); + if (lastErrorsHolder != null) lastErrorsHolder.close(); } public String connectAndGetLogMessages() throws InterruptedException, IOException { @@ -53,7 +53,8 @@ public class LastErrorsHolderTestCase { Instant start = Instant.now(); while (Instant.now().isBefore(start.plus(Duration.ofMinutes(1)))) { try { - socket = SocketChannel.open(new InetSocketAddress(lastErrorsHolderPort)); + InetSocketAddress address = new InetSocketAddress("localhost", lastErrorsHolderPort); + socket = SocketChannel.open(address); break; } catch (Exception e) { Thread.sleep(100); @@ -76,9 +77,12 @@ public class LastErrorsHolderTestCase { @Test public void testLastErrorsHolder() throws IOException, InvalidLogFormatException, InterruptedException { - LastErrorsHolder.Message logMessage1 = new LastErrorsHolder.Message(1433996283, "host1.yahoo.com", "container", LogLevel.ERROR.getName(), "foo"); - LastErrorsHolder.Message logMessage2 = new LastErrorsHolder.Message(1433996284, "host2.yahoo.com", "container", LogLevel.ERROR.getName(), "bar"); - LastErrorsHolder.Message logMessage3 = new LastErrorsHolder.Message(1433996285, "host2.yahoo.com", "container", LogLevel.INFO.getName(), "bar"); + LastErrorsHolder.Message logMessage1 = new LastErrorsHolder.Message(1433996283, "host1.yahoo.com", "container", LogLevel.ERROR + .getName(), "foo"); + LastErrorsHolder.Message logMessage2 = new LastErrorsHolder.Message(1433996284, "host2.yahoo.com", "container", LogLevel.ERROR + .getName(), "bar"); + LastErrorsHolder.Message logMessage3 = new LastErrorsHolder.Message(1433996285, "host2.yahoo.com", "container", LogLevel.INFO + .getName(), "bar"); LastErrorsHolder.Messages messages = new LastErrorsHolder.Messages(); @@ -106,11 +110,13 @@ public class LastErrorsHolderTestCase { } private LogMessage createLogMessage(LastErrorsHolder.Message message) throws InvalidLogFormatException { - return createLogMessage(message.getTime(), message.getHostname(), message.getService(), message.getLogLevel(), message.getMessage()); + return createLogMessage(message.getTime(), message.getHostname(), message.getService(), message.getLogLevel(), message + .getMessage()); } private LogMessage createLogMessage(long time, String hostname, String service, String logLevel, String message) throws InvalidLogFormatException { - return LogMessage.parseNativeFormat(String.format("%d\t%s\t1/1\t%s\tcomponent\t%s\t%s", time, hostname, service, logLevel.toLowerCase(), message)); + return LogMessage.parseNativeFormat(String.format("%d\t%s\t1/1\t%s\tcomponent\t%s\t%s", time, hostname, service, logLevel + .toLowerCase(), message)); } } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java index 4aece1a6b32..3aaa98bae15 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/logmetrics/test/LogMetricsTestCase.java @@ -10,97 +10,97 @@ import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; import com.yahoo.plugin.SystemPropertyConfig; import org.junit.*; + import static org.junit.Assert.*; /** * @author hmusum */ public class LogMetricsTestCase { - // Some of the tests depend upon the number of messages for a - // host, log level etc. to succeed, so you may have update the - // tests if you change something in mStrings. config, debug and - // spam are filtered out and not handled. - private static final String[] mStrings = { - "1095159244.095\thostA\t1/2\tservice\tcomponent\tconfig\tpayload1", - "1095206399.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload2", - "1095206400.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload3", - "1095206401.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload4", - "1095206402.000\thostA\t1/2\tservice\tcomponent\twarning\tpayload5", - "1095206403.000\thostA\t1/2\tservice\tcomponent\terror\tpayload6", - "1095206404.000\thostB\t1/2\tservice\tcomponent\tinfo\tpayload7", - "1095206405.000\thostB\t1/2\tservice\tcomponent\tfatal\tpayload8", - "1095206406.000\thostB\t1/2\tservice\tcomponent\tdebug\tpayload9", - }; - - private static final LogMessage[] msg = new LogMessage[mStrings.length]; - - static { - try { - for (int i = 0; i < mStrings.length; i++) { - msg[i] = LogMessage.parseNativeFormat(mStrings[i]); - } - } catch (InvalidLogFormatException e) { - throw new RuntimeException(e); - } - } - - /** - * Log some messages to the handler and verify that they are - * counted by the handler and that the count for each level is - * correct. - */ + // Some of the tests depend upon the number of messages for a + // host, log level etc. to succeed, so you may have update the + // tests if you change something in mStrings. config, debug and + // spam are filtered out and not handled. + private static final String[] mStrings = { + "1095159244.095\thostA\t1/2\tservice\tcomponent\tconfig\tpayload1", + "1095206399.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload2", + "1095206400.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload3", + "1095206401.000\thostA\t1/2\tservice\tcomponent\tinfo\tpayload4", + "1095206402.000\thostA\t1/2\tservice\tcomponent\twarning\tpayload5", + "1095206403.000\thostA\t1/2\tservice\tcomponent\terror\tpayload6", + "1095206404.000\thostB\t1/2\tservice\tcomponent\tinfo\tpayload7", + "1095206405.000\thostB\t1/2\tservice\tcomponent\tfatal\tpayload8", + "1095206406.000\thostB\t1/2\tservice\tcomponent\tdebug\tpayload9", + }; + + private static final LogMessage[] msg = new LogMessage[mStrings.length]; + + static { + try { + for (int i = 0; i < mStrings.length; i++) { + msg[i] = LogMessage.parseNativeFormat(mStrings[i]); + } + } catch (InvalidLogFormatException e) { + throw new RuntimeException(e); + } + } + + /** + * Log some messages to the handler and verify that they are + * counted by the handler and that the count for each level is + * correct. + */ @Test - public void testLevelCountTotal () throws java.io.IOException, InvalidLogFormatException { - LogMetricsHandler a = new LogMetricsHandler(); + public void testLevelCountTotal() throws java.io.IOException, InvalidLogFormatException { + LogMetricsHandler a = new LogMetricsHandler(); for (LogMessage aMsg : msg) { a.handle(aMsg); } - long count = a.getMetricsCount(); - a.close(); - // Not all messages are processes (debug and spam) - assertEquals(count, 7); - } - + long count = a.getMetricsCount(); + a.close(); + // Not all messages are processes (debug and spam) + assertEquals(count, 7); + } - /** - * Log some messages to the handler and verify that they are - * counted by the handler and that the count for each level is - * correct (the count for each host is summed into one count for - * each level). - */ + /** + * Log some messages to the handler and verify that they are + * counted by the handler and that the count for each level is + * correct (the count for each host is summed into one count for + * each level). + */ @Test - public void testLevelCountAggregated () throws java.io.IOException, InvalidLogFormatException { - LogMetricsHandler a = new LogMetricsHandler(); + public void testLevelCountAggregated() throws java.io.IOException, InvalidLogFormatException { + LogMetricsHandler a = new LogMetricsHandler(); for (LogMessage aMsg : msg) { a.handle(aMsg); } - Map<String, Long> levelCount = a.getMetricsPerLevel(); - assertEquals(levelCount.entrySet().size(), 5); - for (Map.Entry<String, Long> entry : levelCount.entrySet()) { - String key = entry.getKey(); - if (key.equals("config")) { - assertEquals(entry.getValue(), new Long(1)); - } else if (key.equals("info")) { - assertEquals(entry.getValue(), new Long(4)); - } else if (key.equals("warning")) { - assertEquals(entry.getValue(), new Long(1)); - } else if (key.equals("severe")) { - assertEquals(entry.getValue(), new Long(0)); - } else if (key.equals("error")) { - assertEquals(entry.getValue(), new Long(1)); - } else if (key.equals("fatal")) { - assertEquals(entry.getValue(), new Long(1)); - } else if (key.equals("debug")) { - assertEquals(entry.getValue(), new Long(0)); // always 0 - } - } - a.close(); - } + Map<String, Long> levelCount = a.getMetricsPerLevel(); + assertEquals(levelCount.entrySet().size(), 5); + for (Map.Entry<String, Long> entry : levelCount.entrySet()) { + String key = entry.getKey(); + if (key.equals("config")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("info")) { + assertEquals(entry.getValue(), new Long(4)); + } else if (key.equals("warning")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("severe")) { + assertEquals(entry.getValue(), new Long(0)); + } else if (key.equals("error")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("fatal")) { + assertEquals(entry.getValue(), new Long(1)); + } else if (key.equals("debug")) { + assertEquals(entry.getValue(), new Long(0)); // always 0 + } + } + a.close(); + } @Test public void testLogMetricsPlugin() { @@ -118,5 +118,5 @@ public class LogMetricsTestCase { } lp.shutdownPlugin(); } - + } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java index b0d846a2965..333f0ed1095 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/FormattedBufferCacheTestCase.java @@ -7,6 +7,7 @@ package com.yahoo.logserver.handlers.replicator; import java.nio.ByteBuffer; + import com.yahoo.logserver.handlers.replicator.FormattedBufferCache; import com.yahoo.log.LogMessage; import com.yahoo.logserver.formatter.LogFormatter; @@ -14,17 +15,18 @@ import com.yahoo.logserver.formatter.LogFormatterManager; import com.yahoo.logserver.test.MockLogEntries; import org.junit.*; + import static org.junit.Assert.*; /** * Unit tests for thge LogMessage class. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class FormattedBufferCacheTestCase { @Test - public void testCache () { + public void testCache() { LogMessage msgs[] = MockLogEntries.getMessages(); FormattedBufferCache cache = new FormattedBufferCache(); String n[] = LogFormatterManager.getFormatterNames(); diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java index dbb0b7817f6..e10349c7414 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorPluginTestCase.java @@ -12,19 +12,19 @@ import com.yahoo.plugin.SystemPropertyConfig; public class ReplicatorPluginTestCase { ReplicatorPlugin rp; - + @Before public void setUp() { rp = new ReplicatorPlugin(); } - + @After public void tearDown() { - if (rp!=null) { + if (rp != null) { rp.shutdownPlugin(); } } - + @Test public void testReplicatorPlugin() { System.setProperty("replicatorplugin.test.port", "18325"); diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java index 37f25e47a1c..7c884c6e0a3 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/replicator/ReplicatorTestCase.java @@ -1,18 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.replicator; -import static org.junit.Assert.*; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; - -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - import com.yahoo.io.Listener; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogLevel; @@ -21,6 +9,18 @@ import com.yahoo.logserver.Server; import com.yahoo.logserver.filter.LogFilter; import com.yahoo.logserver.filter.LogFilterManager; import com.yahoo.logserver.formatter.LogFormatterManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class ReplicatorTestCase { Server server; @@ -40,7 +40,7 @@ public class ReplicatorTestCase { long timeout = 60000; while (System.currentTimeMillis() < (start + timeout)) { try { - socket = SocketChannel.open(new InetSocketAddress(18321)); + socket = SocketChannel.open(new InetSocketAddress("localhost", 18321)); break; } catch (Exception e) { Thread.sleep(100); @@ -53,21 +53,21 @@ public class ReplicatorTestCase { @After public void tearDown() throws InterruptedException { - if (serverThread!=null) { + if (serverThread != null) { serverThread.interrupt(); serverThread.join(); } - if (replicator!=null) replicator.close(); + if (replicator != null) replicator.close(); } @Test public void testReplicator() throws IOException, InvalidLogFormatException { LogMessage msg = LogMessage. - parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tevent\tfoo"); + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tevent\tfoo"); assertFalse(conn.isLoggable(msg)); // Default all muted conn.onUse("system.all"); assertTrue(conn.isLoggable(msg)); - assertTrue(conn.getTotalBytesWritten()>50); // Should be in this ballpark + assertTrue(conn.getTotalBytesWritten() > 50); // Should be in this ballpark conn.onCommand("use system.mute"); assertFalse(conn.isLoggable(msg)); assertEquals("system.mute", conn.getLogFilterName()); @@ -87,7 +87,7 @@ public class ReplicatorTestCase { assertEquals(22, conn.getNumHandled()); // 4 formatters conn.onCommand("listformatters"); assertEquals(26, conn.getNumHandled()); // 4 formatters - + conn.onStats(); assertEquals(27, conn.getNumHandled()); // 1 line conn.onCommand("stats"); @@ -113,21 +113,21 @@ public class ReplicatorTestCase { }); conn.onUse("test.onlyerror"); assertFalse(conn.isLoggable(LogMessage. - parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tdebug\tfoo"))); + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\tdebug\tfoo"))); assertTrue(conn.isLoggable(LogMessage. - parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); assertEquals(conn.selectOps(), 1); assertEquals(conn.description(), "Only error"); conn.setFilter(null); assertTrue(conn.isLoggable(LogMessage. - parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); assertEquals(conn.description(), "No filter defined"); - assertEquals(conn.getRemoteHost(), "0.0.0.0"); + assertEquals(conn.getRemoteHost(), "localhost"); conn.onFormatter("nonexistant"); assertEquals(conn.formatter, LogFormatterManager.getLogFormatter("system.nullformatter")); // unchanged conn.onUse("nonexistant"); assertTrue(conn.isLoggable(LogMessage. - parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); + parseNativeFormat("1343996283.239582\texample.yahoo.com\t27301/7637\tconfig-sentinel\trunserver\terror\tbar"))); conn.close(); } diff --git a/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore b/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/httphandlers/.gitignore +++ /dev/null diff --git a/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java index 5f0897187c4..d81db60473c 100644 --- a/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/net/control/test/LevelsTestCase.java @@ -5,10 +5,11 @@ import com.yahoo.logserver.net.control.Levels; import com.yahoo.logserver.net.control.State; import org.junit.*; + import static org.junit.Assert.*; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LevelsTestCase { /** @@ -29,10 +30,9 @@ public class LevelsTestCase { * which has been deemed to be "reasonable". In general, it will * specify forwarding all log levels except "debug" and it will * specify "spam" to be turned off. - * */ @Test - public void testDefaultLevels () { + public void testDefaultLevels() { Levels levels = new Levels(); assertSame(State.FORWARD, levels.getLevelState("event")); assertSame(State.FORWARD, levels.getLevelState("fatal")); @@ -47,7 +47,6 @@ public class LevelsTestCase { /** * This test also documents/verifies the default behavior * of the Levels class. - * */ @Test public void testToString() { @@ -62,20 +61,20 @@ public class LevelsTestCase { * Clone testing */ @Test - public void testClone () { + public void testClone() { Levels l1 = Levels.parse("error=noforward"); assertEquals(l1.toString(), l1.clone().toString()); assertSame(State.NOFORWARD, l1.getLevelState("error")); - assertSame(State.NOFORWARD, ((Levels)l1.clone()).getLevelState("error")); + assertSame(State.NOFORWARD, ((Levels) l1.clone()).getLevelState("error")); assertSame(l1.getLevelState("error"), - ((Levels)l1.clone()).getLevelState("error")); + ((Levels) l1.clone()).getLevelState("error")); } /** * test parser */ @Test - public void testUpdateLevels () { + public void testUpdateLevels() { Levels l1 = Levels.parse("error=noforward"); assertSame(State.NOFORWARD, l1.getLevelState("error")); diff --git a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java index 90835930fc0..077cef2d543 100644 --- a/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/net/test/LogConnectionTestCase.java @@ -27,7 +27,7 @@ import static org.junit.Assert.*; * Unit tests for the log connection class. Wow is this too * complex! * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogConnectionTestCase { private static final Logger log = Logger.getLogger(LogConnectionTestCase.class.getName()); diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java index 37ad625247d..eba3508253d 100644 --- a/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/test/LogDispatcherTestCase.java @@ -10,48 +10,51 @@ import com.yahoo.log.LogMessage; import com.yahoo.logserver.handlers.LogHandler; import org.junit.*; + import static org.junit.Assert.*; /** * Unit tests for thge LogMessage class. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogDispatcherTestCase { private static LogMessage sample1; private static LogMessage sample2; + static { try { sample1 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info Starting up, called as ./log/logtest"); sample2 = LogMessage.parseNativeFormat("1096639280.524133 malfunction 26851 - logtest info More crap"); - } - catch (InvalidLogFormatException e) { + } catch (InvalidLogFormatException e) { assertTrue(false); } } public static class MockHandler implements LogHandler { - public final List<LogMessage> messages= new ArrayList<LogMessage>(5); + public final List<LogMessage> messages = new ArrayList<LogMessage>(5); public int flushCalled = 0; public int closeCalled = 0; - public void handle (LogMessage msg) { + public void handle(LogMessage msg) { messages.add(msg); } - public void handle (List<LogMessage> messages) { - for (LogMessage lm : messages) { - handle(lm); - } + public void handle(List<LogMessage> messages) { + for (LogMessage lm : messages) { + handle(lm); + } } - public void flush () { + public void flush() { flushCalled++; } - public void close () { + + public void close() { closeCalled++; } - public String getName () { + + public String getName() { return MockHandler.class.getName(); } } @@ -73,7 +76,7 @@ public class LogDispatcherTestCase { } @Test - public void testTestLogHandlerRegistration () { + public void testTestLogHandlerRegistration() { MockHandler handler = new MockHandler(); LogDispatcher dispatcher = new LogDispatcher(); dispatcher.registerLogHandler(handler); @@ -86,7 +89,7 @@ public class LogDispatcherTestCase { } @Test - public void testMessageCount () { + public void testMessageCount() { MockHandler handler = new MockHandler(); LogDispatcher dispatcher = new LogDispatcher(); dispatcher.registerLogHandler(handler); @@ -102,7 +105,7 @@ public class LogDispatcherTestCase { } @Test - public void testVerifyMessages () { + public void testVerifyMessages() { MockHandler handler = new MockHandler(); LogDispatcher dispatcher = new LogDispatcher(); dispatcher.registerLogHandler(handler); @@ -116,7 +119,7 @@ public class LogDispatcherTestCase { // TODO: this test makes very little sense until we refactor a bit @Test - public void testClose () { + public void testClose() { MockHandler handler = new MockHandler(); LogDispatcher dispatcher = new LogDispatcher(); dispatcher.registerLogHandler(handler); diff --git a/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java index a9bb8084a7c..97f3d22e620 100644 --- a/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/test/LogMessageTestCase.java @@ -16,9 +16,9 @@ import org.junit.*; import static org.junit.Assert.*; /** - * Unit tests for thge LogMessage class. + * Unit tests for the LogMessage class. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogMessageTestCase { diff --git a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java index 18cf8afb8a0..9912192b850 100644 --- a/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java +++ b/logserver/src/test/java/com/yahoo/logserver/test/MockLogEntries.java @@ -14,7 +14,7 @@ import java.util.List; * This class is used to gain access to a bunch of log entries * so we can use the same log messages in several different tests * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class MockLogEntries { private static final MockLogEntries instance = new MockLogEntries(); @@ -25,7 +25,7 @@ public class MockLogEntries { * Private constructor which reads the log messages and builds * an array of LogMessage entries. */ - private MockLogEntries () { + private MockLogEntries() { List<LogMessage> msgs = new LinkedList<LogMessage>(); try { String name = "src/test/files/logEntries.txt"; @@ -35,7 +35,7 @@ public class MockLogEntries { msgs.add(m); } } catch (InvalidLogFormatException | IOException e) { - // do nothing + // do nothing } LogMessage[] m = new LogMessage[msgs.size()]; diff --git a/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp b/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp index d7478c4755a..136d5dbd4ba 100644 --- a/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp +++ b/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/fastlib/text/normwordfolder.h> +#include <cassert> #include <fstream> #include <iostream> diff --git a/lowercasing_test/src/tests/lowercasing/casingvariants_vespalib.cpp b/lowercasing_test/src/tests/lowercasing/casingvariants_vespalib.cpp index d6df9686e86..fcbc3b701e1 100644 --- a/lowercasing_test/src/tests/lowercasing/casingvariants_vespalib.cpp +++ b/lowercasing_test/src/tests/lowercasing/casingvariants_vespalib.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/vespalib/text/utf8.h> #include <fstream> #include <iostream> +#include <cassert> using vespalib::LowerCase; using vespalib::Utf8ReaderForZTS; diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml new file mode 100644 index 00000000000..ab201932882 --- /dev/null +++ b/maven-plugins/pom.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugins</artifactId> + <name>bundle-plugins</name> + <packaging>pom</packaging> + <version>6-SNAPSHOT</version> + <description>Parent artifact for Vespa maven plugins.</description> + <url>http://yahoo.github.io/vespa</url> + + <modules> + <module>../annotations</module> + <module>../scalalib</module> + <module>../bundle-plugin</module> + <module>../configgen</module> + <module>../config-class-plugin</module> + </modules> +</project> diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp index f5e16032361..50a4e8ef56c 100644 --- a/memfilepersistence/src/tests/device/devicemanagertest.cpp +++ b/memfilepersistence/src/tests/device/devicemanagertest.cpp @@ -3,6 +3,7 @@ #include <vespa/memfilepersistence/device/devicemanager.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> namespace storage { diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/environment.h b/memfilepersistence/src/vespa/memfilepersistence/common/environment.h index 0f81d61a943..93ef0768148 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/environment.h +++ b/memfilepersistence/src/vespa/memfilepersistence/common/environment.h @@ -18,7 +18,6 @@ #include "config_lock_guard.h" #include "config_aliases.h" #include <vespa/memfilepersistence/device/mountpointlist.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/config/helper/configfetcher.h> diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp index 651543219ec..63e0f35c95c 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "options.h" -#include <vespa/config-stor-memfilepersistence.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.slotfile.options"); -namespace storage { - -namespace memfile { +namespace storage::memfile { Options::Options(const vespa::config::storage::StorMemfilepersistenceConfig& newConfig, const vespa::config::content::PersistenceConfig& newPersistenceConfig) @@ -179,5 +177,3 @@ void Options::print(std::ostream& out, bool verbose, } } - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.h b/memfilepersistence/src/vespa/memfilepersistence/common/options.h index 6199bf59f37..29e46b29dce 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/options.h +++ b/memfilepersistence/src/vespa/memfilepersistence/common/options.h @@ -17,15 +17,13 @@ #pragma once -#include <vespa/storageframework/storageframework.h> #include <vespa/config-stor-memfilepersistence.h> #include <vespa/config-persistence.h> #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/storageframework/generic/clock/time.h> -namespace storage { - -namespace memfile { +namespace storage::memfile { struct Options : public vespalib::Printable { @@ -128,6 +126,3 @@ struct Options : public vespalib::Printable }; } - -} - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp index f02423cb194..2bc85fbc422 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "types.h" -#include <sstream> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> +#include <sstream> -namespace storage { -namespace memfile { +namespace storage::memfile { const framework::MicroSecTime Types::MAX_TIMESTAMP(framework::MicroSecTime::max()); const framework::MicroSecTime Types::UNSET_TIMESTAMP(0); @@ -34,5 +34,17 @@ operator<<(std::ostream& os, const DataLocation& loc) return os; } -} // memfile -} // storage +const char* +Types::getMemFileFlagName(MemFileFlag flag) { + switch (flag) { + case FILE_EXIST: return "FILE_EXIST"; + case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ"; + case BODY_BLOCK_READ: return "BODY_BLOCK_READ"; + case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED"; + case SLOTS_ALTERED: return "SLOTS_ALTERED"; + case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag + default: return "INVALID"; + } +} + +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.h b/memfilepersistence/src/vespa/memfilepersistence/common/types.h index ab6ae1e52f8..c1f29571037 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/common/types.h +++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.h @@ -16,12 +16,11 @@ #pragma once -#include <vespa/storageframework/storageframework.h> #include <vespa/persistence/spi/bucketinfo.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/storageframework/generic/clock/time.h> -namespace storage { -namespace memfile { +namespace storage::memfile { /** * \class storage::slotfile::DataLocation @@ -174,25 +173,11 @@ struct Types { } } - static const char* getMemFileFlagName(MemFileFlag flag) { - switch (flag) { - case FILE_EXIST: return "FILE_EXIST"; - case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ"; - case BODY_BLOCK_READ: return "BODY_BLOCK_READ"; - case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED"; - case SLOTS_ALTERED: return "SLOTS_ALTERED"; - case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag - default: return "INVALID"; - } - } - - static void verifyLegalFlags(uint32_t flags, uint32_t legal, - const char* operation); + static const char* getMemFileFlagName(MemFileFlag flag); + static void verifyLegalFlags(uint32_t flags, uint32_t legal, const char* operation); protected: ~Types() {} // Noone should refer to objects as Types objects }; -} // memfile -} // storage - +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp index 2ae27f06671..ba249df2b73 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp @@ -2,6 +2,7 @@ #include "devicemanager.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/xmlstream.h> namespace storage::memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp index 8ddf215d616..fdecb38ccf0 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp @@ -5,13 +5,12 @@ #include <vespa/vespalib/util/exceptions.h> #include <fstream> #include <sstream> +#include <sys/stat.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.devicemapper"); -namespace storage { - -namespace memfile { +namespace storage::memfile { namespace { uint64_t getDevice(const std::string& path) { @@ -95,5 +94,3 @@ AdvancedDeviceMapper::getPartitionId(const std::string& fileOnFS) const } } - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp index b3751cddbed..614d399ca22 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp @@ -7,14 +7,14 @@ #include <vespa/config/helper/configfetcher.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/guard.h> +#include <vespa/vespalib/text/stringtokenizer.h> #include <fstream> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.mountpointlist"); -namespace storage { - -namespace memfile { +namespace storage::memfile { using vespalib::getLastErrorString; using vespalib::DirPointer; @@ -642,5 +642,3 @@ MountPointList::reportXmlStatus(vespalib::xml::XmlOutputStream& xos, } } - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h index f80829e8aed..08adb66f9f1 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h +++ b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h @@ -18,7 +18,7 @@ #include "devicemanager.h" #include "directory.h" -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/vespalib/util/printable.h> diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp index 3bd855553c7..9ff402a89b5 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp @@ -3,13 +3,12 @@ #include "partition.h" #include "devicemanager.h" #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.device.partition"); -namespace storage { - -namespace memfile { +namespace storage::memfile { Partition::Partition(DeviceManager& manager, uint64_t id, @@ -59,6 +58,4 @@ Partition::print(std::ostream& out, bool verbose, Device::print(out, verbose, indent); } -} // memfile - -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp index b5b94d29336..ebc33d40f42 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp @@ -3,13 +3,13 @@ #include "partitionmonitor.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".persistence.device.partition.monitor"); -namespace storage { - -namespace memfile { +namespace storage::memfile { namespace { @@ -387,5 +387,3 @@ PartitionMonitor::overrideRealStat(uint32_t blockSize, uint32_t totalBlocks, } } - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp index 3a83cee392a..e5466634283 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp @@ -5,11 +5,10 @@ #include <vespa/vespalib/util/exceptions.h> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.filescanner"); -namespace storage { -namespace memfile { +namespace storage::memfile { FileScanner::Metrics::Metrics(framework::Clock& clock) : metrics::MetricSet("dbinit.filescan", "", @@ -238,5 +237,4 @@ FileScanner::handleBadLocation(const document::BucketId& bucket, return true; } -} // memfile -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.h b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.h index ff814eed19e..7d06e442400 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.h +++ b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.h @@ -13,15 +13,15 @@ #include <vespa/memfilepersistence/device/mountpointlist.h> #include <vespa/memfilepersistence/mapper/bucketdirectorymapper.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/metrics/metrics.h> +#include <vespa/storageframework/generic/component/component.h> +#include <vespa/storageframework/generic/clock/timer.h> namespace document { class BucketId; } -namespace storage { -namespace memfile { +namespace storage::memfile { class FileScanner : private framework::Component { public: @@ -102,6 +102,4 @@ private: std::vector<uint32_t>& path); }; -} // memfile -} // storage - +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp index 65b92e212f5..175c156dc9d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp @@ -4,8 +4,9 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.bufferedfilewriter"); namespace storage { diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.h index d56f5f80441..55e815d88c3 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.h @@ -12,9 +12,9 @@ #pragma once #include <vespa/memfilepersistence/common/types.h> +#include <vespa/vespalib/util/printable.h> -namespace storage { -namespace memfile { +namespace storage::memfile { class MemSlot; @@ -53,10 +53,7 @@ private: const std::vector<DataLocation>& desiredLocations, uint32_t maxGap); - void scheduleLocation(DataLocation loc, - std::vector<DataLocation>&); + void scheduleLocation(DataLocation loc, std::vector<DataLocation>&); }; -} // memfile -} // storage - +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp index d9cc0c35cc4..950acf09b28 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp @@ -8,12 +8,13 @@ #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/storageframework/generic/clock/timer.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfilev1"); -namespace storage { -namespace memfile { +namespace storage::memfile { namespace { @@ -1023,4 +1024,3 @@ MemFileV1Serializer::verify(MemFile& file, Environment& env, } } -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp index 4ba25a48950..66f6cdff083 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp @@ -3,15 +3,15 @@ #include "memfile_v1_verifier.h" #include "memfilemapper.h" #include "simplememfileiobuffer.h" +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.hpp> -#include <vespa/log/log.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".persistence.memfilev1.verifier"); -namespace storage { - -namespace memfile { +namespace storage::memfile { namespace { @@ -690,8 +690,4 @@ MemFileV1Verifier::verifyUniqueTimestamps( okSlots.swap(slots); } - } - -} - diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp index 4888154d3bb..2d91e46c7b2 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp @@ -6,12 +6,13 @@ #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h> #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/vdslib/distribution/distribution.h> +#include <vespa/storageframework/generic/clock/timer.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.mapper"); -namespace storage { -namespace memfile { +namespace storage::memfile { // Repair defined in macro, such that log entries will be unique for the various // instances calling it (different file line numbers) @@ -335,5 +336,4 @@ MemFileMapper::removeAllSlotsExcept(MemFile& file, std::vector<Timestamp>& keep) file.removeSlots(slotsToRemove); } -} // storage -} // memfile +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp index 1f5344d466e..ef5c865eb5a 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp @@ -2,10 +2,13 @@ #include "simplememfileiobuffer.h" #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/memfilepersistence/common/environment.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <vespa/document/util/bytebuffer.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".memfile.simpleiobuffer"); namespace storage::memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp index 6b3f72d9187..73defc043af 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp @@ -12,7 +12,7 @@ #include <ext/algorithm> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.memfile.memfile"); namespace { diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h index 112c5e6c399..7b34b2eba9d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h @@ -22,10 +22,10 @@ #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> #include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/generic/component/component.h> -namespace storage { -namespace memfile { +namespace storage::memfile { class MemFilePersistenceCacheMetrics; class Environment; // Avoid cyclic dependency with environment @@ -298,6 +298,4 @@ public: void printCacheEntriesHtml(std::ostream& out) const; }; -} // storage -} // memfile - +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h index e0b1a7b9a2a..42d3fb0b238 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h +++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/memfilepersistence/common/types.h> +#include <map> namespace storage { namespace memfile { diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp index a6b5b5bd782..6cb1c523325 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp @@ -4,12 +4,12 @@ #include "visitorslotmatcher.h" #include "cacheevictionguard.h" #include <vespa/document/select/bodyfielddetector.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.handler.iterator"); -namespace storage { -namespace memfile { +namespace storage::memfile { CachePrefetchRequirements CachePrefetchRequirements::createFromSelection(const document::DocumentTypeRepo& repo, @@ -435,4 +435,3 @@ IteratorHandler::iterate(spi::IteratorId id, uint64_t maxByteSize) } } -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp index 1f82ffb2172..0fc2aa2ece6 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp @@ -3,6 +3,7 @@ #include "joinoperationhandler.h" #include "cacheevictionguard.h" #include <vespa/memfilepersistence/mapper/memfilemapper.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.handler.join"); diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp index a4f10688feb..9e580d4e234 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp @@ -3,10 +3,10 @@ #include "memfilepersistenceprovider.h" #include <vespa/memfilepersistence/common/exceptions.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/config/helper/configgetter.hpp> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".memfilepersistenceprovider"); @@ -28,8 +28,7 @@ if ((context).getTrace().shouldTrace(9)) { \ (context).getTrace().trace(9, messageToTrace); \ } -namespace storage { -namespace memfile { +namespace storage::memfile { namespace { @@ -888,5 +887,4 @@ MemFilePersistenceProvider::reportStatus(std::ostream& out, return true; } -} // memfile -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.h b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.h index 724a77ec443..52e10737f33 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.h +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.h @@ -12,7 +12,6 @@ #include <vespa/memfilepersistence/mapper/memfilemapper.h> #include <vespa/memfilepersistence/init/filescanner.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storageframework/generic/status/httpurlpath.h> #include <vespa/config/config.h> diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp index da085ca9137..c9972dbd02b 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp @@ -3,12 +3,12 @@ #include "operationhandler.h" #include <vespa/memfilepersistence/common/exceptions.h> #include <vespa/document/select/parser.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".persistence.memfile.handler.operation"); -namespace storage { -namespace memfile { +namespace storage::memfile { OperationHandler::OperationHandler(Environment& env) : _env(env) @@ -283,5 +283,4 @@ OperationHandler::parseDocumentSelection( return ret; } -} // memfile -} // storage +} diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore b/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore index c7687cb62d2..5b3727b1f5e 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore @@ -1,5 +1,4 @@ /.depend /Makefile -/dumpslotfile -/vdsdisktool -vdsdisktool-bin +vespa-dump-slotfile +vespa-vds-disktool-bin diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt index cc8ad0a2320..dabec5b6a8e 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt +++ b/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt @@ -8,7 +8,7 @@ vespa_add_library(memfilepersistence_tools OBJECT vespa_add_executable(memfilepersistence_dumpslotfile_app SOURCES dumpslotfileapp.cpp - OUTPUT_NAME dumpslotfile + OUTPUT_NAME vespa-dump-slotfile INSTALL bin DEPENDS memfilepersistence @@ -16,7 +16,7 @@ vespa_add_executable(memfilepersistence_dumpslotfile_app vespa_add_executable(memfilepersistence_vdsdisktool_app SOURCES vdsdiskapp.cpp - OUTPUT_NAME vdsdisktool-bin + OUTPUT_NAME vespa-vds-disktool-bin INSTALL bin DEPENDS memfilepersistence diff --git a/messagebus-disc/OWNERS b/messagebus-disc/OWNERS index 123437e2758..12b533ec610 100644 --- a/messagebus-disc/OWNERS +++ b/messagebus-disc/OWNERS @@ -1 +1 @@ -dybis +havardpe diff --git a/messagebus/OWNERS b/messagebus/OWNERS index 46254c9df7c..816b7b14767 100644 --- a/messagebus/OWNERS +++ b/messagebus/OWNERS @@ -1,2 +1 @@ -dybis baldersheim
\ No newline at end of file diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Reply.java b/messagebus/src/main/java/com/yahoo/messagebus/Reply.java index ba184bb38ad..38ef2c43937 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Reply.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Reply.java @@ -2,7 +2,9 @@ package com.yahoo.messagebus; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Stream; /** @@ -138,4 +140,16 @@ public abstract class Reply extends Routable { public Stream<Error> getErrors() { return errors.stream(); } + + /** + * Retrieves a set of integer error codes + */ + public Set<Integer> getErrorCodes() { + Set<Integer> errorCodes = new HashSet<>(); + for (Error error : errors) { + errorCodes.add(error.getCode()); + } + return errorCodes; + } + } diff --git a/messagebus/src/vespa/messagebus/callstack.cpp b/messagebus/src/vespa/messagebus/callstack.cpp index 5311a75c3c5..44ae4190698 100644 --- a/messagebus/src/vespa/messagebus/callstack.cpp +++ b/messagebus/src/vespa/messagebus/callstack.cpp @@ -4,6 +4,7 @@ #include "message.h" #include "reply.h" #include "idiscardhandler.h" +#include <cassert> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 06cbef7b57b..2991c44c9d6 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -8,8 +8,10 @@ #include <vespa/messagebus/routing/routingnode.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/sbmirror.h> -#include <vespa/log/log.h> +#include <vespa/vespalib/component/vtag.h> +#include <vespa/fnet/scheduler.h> +#include <vespa/log/log.h> LOG_SETUP(".rpcnetwork"); namespace { diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.h b/messagebus/src/vespa/messagebus/rpcmessagebus.h index de120b55e62..9bd290c62d8 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.h +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.h @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/messagebus/network/rpcnetwork.h> -#include <string> -#include <vespa/config/helper/legacysubscriber.h> #include "messagebus.h" #include "configagent.h" #include "protocolset.h" +#include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/config/helper/legacysubscriber.h> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/testlib/oosserver.h b/messagebus/src/vespa/messagebus/testlib/oosserver.h index a9ab128aa5d..6d067326bec 100644 --- a/messagebus/src/vespa/messagebus/testlib/oosserver.h +++ b/messagebus/src/vespa/messagebus/testlib/oosserver.h @@ -3,8 +3,8 @@ #pragma once #include <vespa/vespalib/util/sync.h> -#include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/sbregister.h> +#include <vespa/fnet/frt/supervisor.h> #include <string> #include <vector> #include "oosstate.h" diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp index fa71cb9f9ad..12f53e31764 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp @@ -2,6 +2,9 @@ #include "slobrok.h" #include <vespa/slobrok/server/sbenv.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/transport.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".slobrok"); diff --git a/messagebus_test/OWNERS b/messagebus_test/OWNERS index 7379f3287ad..101fe620dd4 100644 --- a/messagebus_test/OWNERS +++ b/messagebus_test/OWNERS @@ -1,3 +1,2 @@ -dybis baldersheim diff --git a/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp b/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp index e15b57859b7..0aa174ab9b9 100644 --- a/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp +++ b/messagebus_test/src/tests/compile-cpp/compile-cpp.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("compile-cpp_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/messagebus_test/src/tests/error/error.cpp b/messagebus_test/src/tests/error/error.cpp index 749874a5ef2..a04bdddf23f 100644 --- a/messagebus_test/src/tests/error/error.cpp +++ b/messagebus_test/src/tests/error/error.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("error_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/messagebus_test/src/tests/speed/speed.cpp b/messagebus_test/src/tests/speed/speed.cpp index 1cfe1ff5704..5e7128ea1db 100644 --- a/messagebus_test/src/tests/speed/speed.cpp +++ b/messagebus_test/src/tests/speed/speed.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("speed_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/messagebus_test/src/tests/trace/trace.cpp b/messagebus_test/src/tests/trace/trace.cpp index a924d55195f..a69565bacf3 100644 --- a/messagebus_test/src/tests/trace/trace.cpp +++ b/messagebus_test/src/tests/trace/trace.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("trace_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/metrics/src/tests/countmetrictest.cpp b/metrics/src/tests/countmetrictest.cpp index 82eb65caf25..8a26d7d6dc5 100644 --- a/metrics/src/tests/countmetrictest.cpp +++ b/metrics/src/tests/countmetrictest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/objects/floatingpointtype.h> #include <vespa/metrics/countmetric.h> diff --git a/metrics/src/tests/loadmetrictest.cpp b/metrics/src/tests/loadmetrictest.cpp index 97af77169de..b8eac5b4e94 100644 --- a/metrics/src/tests/loadmetrictest.cpp +++ b/metrics/src/tests/loadmetrictest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/metrics/loadmetric.h> #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/loadmetric.hpp> diff --git a/metrics/src/tests/metric_timer_test.cpp b/metrics/src/tests/metric_timer_test.cpp index da5ab0c3dfe..48ba2f932df 100644 --- a/metrics/src/tests/metric_timer_test.cpp +++ b/metrics/src/tests/metric_timer_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/metrictimer.h> diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 0ba6bdfbd2f..e4a2761cd73 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -9,6 +9,7 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> LOG_SETUP(".test.metricmanager"); diff --git a/metrics/src/tests/metrictest.cpp b/metrics/src/tests/metrictest.cpp index 67dd608051a..f232f9b69aa 100644 --- a/metrics/src/tests/metrictest.cpp +++ b/metrics/src/tests/metrictest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/countmetric.h> diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp index 0343174e0b3..b3bb1518ba0 100644 --- a/metrics/src/tests/summetrictest.cpp +++ b/metrics/src/tests/summetrictest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/metrics/metrics.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/metrics/src/vespa/metrics/.gitignore b/metrics/src/vespa/metrics/.gitignore index d6b11486175..1f3106faec5 100644 --- a/metrics/src/vespa/metrics/.gitignore +++ b/metrics/src/vespa/metrics/.gitignore @@ -6,5 +6,4 @@ Makefile config-metricsmanager.h config-metricsmanager.cpp - /libmetrics.so.5.1 diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp index 57ce8e5ffc1..0c99409a8bb 100644 --- a/metrics/src/vespa/metrics/metric.cpp +++ b/metrics/src/vespa/metrics/metric.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <iterator> namespace metrics { diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 572779be855..c352f0ee5ab 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -10,11 +10,12 @@ #include "xmlwriter.h" #include <vespa/config/print/ostreamconfigwriter.h> #include <vespa/vespalib/text/stringtokenizer.h> - +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/log/log.h> +#include <sstream> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".metrics.manager"); namespace metrics { diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp index 15ee22ba1f2..c9acbaafce2 100644 --- a/metrics/src/vespa/metrics/metricset.cpp +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -4,10 +4,10 @@ #include "memoryconsumption.h" #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <list> #include <vespa/log/log.h> - LOG_SETUP(".metrics.metricsset"); namespace metrics { diff --git a/metrics/src/vespa/metrics/printutils.cpp b/metrics/src/vespa/metrics/printutils.cpp index 2373d2391ff..2209ebd5da7 100644 --- a/metrics/src/vespa/metrics/printutils.cpp +++ b/metrics/src/vespa/metrics/printutils.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "printutils.h" +#include <vespa/vespalib/util/stringfmt.h> namespace metrics { namespace printutils { diff --git a/metrics/src/vespa/metrics/summetric.hpp b/metrics/src/vespa/metrics/summetric.hpp index 1a54067b01f..3e85aa0e215 100644 --- a/metrics/src/vespa/metrics/summetric.hpp +++ b/metrics/src/vespa/metrics/summetric.hpp @@ -5,6 +5,7 @@ #include "metricset.h" #include "memoryconsumption.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <ostream> namespace metrics { diff --git a/metrics/src/vespa/metrics/valuemetric.hpp b/metrics/src/vespa/metrics/valuemetric.hpp index 65a0c2bb5b2..b09f35e3272 100644 --- a/metrics/src/vespa/metrics/valuemetric.hpp +++ b/metrics/src/vespa/metrics/valuemetric.hpp @@ -4,6 +4,7 @@ #include "valuemetric.h" #include "memoryconsumption.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <sstream> namespace metrics { diff --git a/metrics/src/vespa/metrics/valuemetricvalues.hpp b/metrics/src/vespa/metrics/valuemetricvalues.hpp index d8becca3e72..e7f3e9e3239 100644 --- a/metrics/src/vespa/metrics/valuemetricvalues.hpp +++ b/metrics/src/vespa/metrics/valuemetricvalues.hpp @@ -3,6 +3,7 @@ #include "valuemetricvalues.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <ostream> namespace metrics { diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp index 66f83cdd4bf..4a998b79294 100644 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ b/metrics/src/vespa/metrics/xmlwriter.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/metrics/xmlwriter.h> - -#include <vespa/metrics/countmetric.h> -#include <vespa/metrics/metricset.h> -#include <vespa/metrics/metricsnapshot.h> -#include <vespa/metrics/valuemetric.h> +#include "xmlwriter.h" +#include "countmetric.h" +#include "metricset.h" +#include "metricsnapshot.h" +#include "valuemetric.h" +#include <vespa/vespalib/util/xmlstream.h> #include <sstream> namespace metrics { diff --git a/node-admin/.gitignore b/node-admin/.gitignore index 67b21d2d89c..5881f4b513b 100644 --- a/node-admin/.gitignore +++ b/node-admin/.gitignore @@ -1 +1 @@ -include/Dockerfile +node-admin-zone-app/components diff --git a/node-admin/README.md b/node-admin/README.md index 7b3e9805d21..8352cbd3c43 100644 --- a/node-admin/README.md +++ b/node-admin/README.md @@ -31,13 +31,13 @@ implements all of the basic methods you need to get started. To start a local zone, simply run: ``` - DockerImage vespaDockerBase = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.53.134"); + DockerImage dockerImage = new DockerImage("docker-registry.ops.yahoo.com:4443/vespa/ci:6.111.21"); + Path pathToVespaRoot = Paths.get("/home/valerijf/dev/vespa"); Path pathToContainerStorage = Paths.get("/home/docker/container-storage"); - - RunVespaLocal runVespaLocal = new RunVespaLocal(); - runVespaLocal.buildVespaLocalImage(vespaDockerBase); - runVespaLocal.startLocalZoneWithNodes(5); - runVespaLocal.startNodeAdminAsContainer(pathToContainerStorage); + + RunVespaLocal runVespaLocal = new RunVespaLocal(pathToVespaRoot); + runVespaLocal.startLocalZoneWithNodes(dockerImage, 5); + runVespaLocal.startNodeAdminAsContainer(dockerImage, pathToContainerStorage); ``` ### Deploying a Local Application @@ -45,7 +45,7 @@ To start a local zone, simply run: Package and deploy vespa application by running: ``` - Path pathToApp = Paths.get("node-admin/basic-search-for-docker"); + Path pathToApp = pathToVespaRoot.resolve("sample-apps/blog-search"); runVespaLocal.deployApplication(pathToApp); ``` If the deployment is successful, the final log entry should be something like diff --git a/node-admin/basic-search-for-docker/music-data-1.json b/node-admin/basic-search-for-docker/music-data-1.json deleted file mode 100644 index 44ce547370e..00000000000 --- a/node-admin/basic-search-for-docker/music-data-1.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "fields": { - "album": "Bad", - "artist": "Michael Jackson", - "title": "Bad", - "year": 1987, - "duration": 247 - } -} diff --git a/node-admin/basic-search-for-docker/music-data-2.json b/node-admin/basic-search-for-docker/music-data-2.json deleted file mode 100644 index c8e2d3c017c..00000000000 --- a/node-admin/basic-search-for-docker/music-data-2.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "fields": { - "album": "Recovery", - "artist": "Eminem", - "title": "So Bad", - "year": 2010 - } -} - diff --git a/node-admin/basic-search-for-docker/pom.xml b/node-admin/basic-search-for-docker/pom.xml deleted file mode 100644 index 749640b5a34..00000000000 --- a/node-admin/basic-search-for-docker/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ -<?xml version="1.0"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.example</groupId> - <artifactId>basic-application</artifactId> - <packaging>container-plugin</packaging> - <version>1.0.1</version> - <name>application</name> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <test.hide>true</test.hide> - <application>docker</application> - <instance>default</instance> - </properties> - - <parent> - <groupId>com.yahoo.vespa.tenant</groupId> - <artifactId>base</artifactId> - <version>RELEASE</version> - </parent> - - <pluginRepositories> - <pluginRepository> - <id>ymaven</id> - <url>http://ymaven.corp.yahoo.com:9999/proximity/repository/public</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </pluginRepository> - <pluginRepository> - <id>maven2-repository.dev.java.net</id> - <url>http://download.java.net/maven/2</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </pluginRepository> - <pluginRepository> - <id>vespa-maven-release</id> - <name>vespa-maven-release</name> - <url>http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local</url> - <releases> - <enabled>true</enabled> - </releases> - <snapshots> - <enabled>false</enabled> - </snapshots> - </pluginRepository> - </pluginRepositories> - <repositories> - <repository> - <id>ymaven</id> - <url>http://ymaven.corp.yahoo.com:9999/proximity/repository/public</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - <repository> - <id>vespa-maven-release</id> - <name>vespa-maven-release</name> - <url>http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local</url> - <releases> - <enabled>true</enabled> - </releases> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/node-admin/basic-search-for-docker/src/main/application/deployment.xml b/node-admin/basic-search-for-docker/src/main/application/deployment.xml deleted file mode 100644 index 0a2397bbca9..00000000000 --- a/node-admin/basic-search-for-docker/src/main/application/deployment.xml +++ /dev/null @@ -1,5 +0,0 @@ -<deployment version='1.0'> - <test /> - <staging /> - <prod/> -</deployment> diff --git a/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd b/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd deleted file mode 100644 index d6bcaac8ae0..00000000000 --- a/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd +++ /dev/null @@ -1,40 +0,0 @@ -search music { - document music { - field artist type string { - indexing: summary | index - } - field artistId type string { - indexing: summary | attribute - } - - field title type string { - indexing: summary | index - } - - field album type string { - indexing: index - } - - field duration type int { - indexing: summary - } - - field year type int { - indexing: summary | attribute - } - - field popularity type int { - indexing: summary | attribute - } - } - - fieldset default { - fields: artist, title, album - } - - rank-profile song inherits default { - first-phase { - expression:nativeRank(artist,title,album) + if(isNan(attribute(popularity)) == 1, 0,attribute(popularity)) - } - } -} diff --git a/node-admin/basic-search-for-docker/src/main/application/services.xml b/node-admin/basic-search-for-docker/src/main/application/services.xml deleted file mode 100644 index cea6861ca91..00000000000 --- a/node-admin/basic-search-for-docker/src/main/application/services.xml +++ /dev/null @@ -1,81 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<services version='1.0' xmlns:deploy="vespa" xmlns:preprocess="properties"> - - <jdisc id='default' version='1.0'> - <!-- See Vespa tutorial - <document-processing> - <chain id="default"> - <documentprocessor id="com.yahoo.example.ExampleDocumentProcessor" bundle="basic-application"/> - </chain> - </document-processing> - --> - - <!-- YCA Access Control - - Enable this to limit access to your application using YCA; we will support http://yo/athens later. - deploy:environment="prod" is needed if you want to enable easier access to other environments. You will not - be able to use YCA access control from an openhouse instance to the hosted Vespa Dev instance. - - The nonyca-request-chain is needed to allow e.g. health checks to bypass YCA - - In <yca.appid.allow> list the igor roles that includes the hosts that should be allowed to access this application - - See http://vespa.corp.yahoo.com/5/documentation/vespa-security.html - - To test this you can use curl, and add the Yahoo-App-Auth header: - -H "Yahoo-App-Auth: `yca-cert-util -show yahoo.namespace.role|cut -d" " -f2`" - - --> - <!-- - <http deploy:environment="prod"> - <filtering> - <request-chain id="yca-request-chain" excludes="com.yahoo.jdisc.http.filter.security.InputValidationFilter" inherits="standard-request-filters"> - <filter bundle="jdisc_http_filters" class="com.yahoo.jdisc.http.filter.security.YCAFilter" id="access-filter"> - <filter-config> - <yca.appid.allow>yahoo.namespace.role</yca.appid.allow> - <yca.allowAny>false</yca.allowAny> - <yca.optional>false</yca.optional> - </filter-config> - </filter> - <binding>http://*/*</binding> - <binding>https://*/*</binding> - </request-chain> - <request-chain excludes="access-filter" id="nonyca-request-chain" inherits="yca-request-chain"> - <binding>http://*/state/*</binding> - <binding>http://*/status.html</binding> - <binding>http://*/ApplicationStatus</binding> - <binding>https://*/state/*</binding> - <binding>https://*/status.html</binding> - <binding>https://*/ApplicationStatus</binding> - </request-chain> - </filtering> - <server id="main-server" port="4080" /> - </http> - --> - - <search> - <chain id='default' inherits='vespa'> - <searcher bundle='basic-application' id='com.yahoo.example.ExampleSearcher'> - <config name='example.message'> - <message>Hello, Vespa! (test build iteration 90)</message> - </config> - </searcher> - </chain> - </search> - <document-api/> - <nodes count='1' flavor='docker' docker-image='vespa-local:latest'/> - </jdisc> - - <content id='music' version='1.0'> - <redundancy>2</redundancy> - <documents> - <document mode='index' type='music'/> - </documents> - <nodes count='1' flavor='docker' docker-image='vespa-local:latest'/> - <engine> - <proton> - <searchable-copies>1</searchable-copies> - </proton> - </engine> - </content> -</services> diff --git a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java deleted file mode 100644 index 907ab355288..00000000000 --- a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentUpdate; - -import java.net.*; -import java.io.*; - -import com.yahoo.document.datatypes.StringFieldValue; -import org.json.*; -import java.util.logging.Logger; - -/** - * A document processor - * - * @author Joe Developer - */ - - -public class ExampleDocumentProcessor extends DocumentProcessor { - - private static final String artistField = "artist"; - private static final String artistIdField = "artistId"; - private static final String spotifyUrl = "https://api.spotify.com/v1/search?type=artist&limit=1&q="; - private static final Logger log = Logger.getLogger(ExampleDocumentProcessor.class.getName()); - - public Progress process(Processing processing) { - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - Document document = ((DocumentPut) op).getDocument(); - addArtistId(document); - } else if (op instanceof DocumentUpdate) { - DocumentUpdate update = (DocumentUpdate) op; - //TODO do something to 'update' here - } else if (op instanceof DocumentRemove) { - DocumentRemove remove = (DocumentRemove) op; - //TODO do something to 'remove' here - } - } - return Progress.DONE; - } - - /** - * Query Spotify API, parse JSON and set Artist ID - * - * @param document a Vespa Document - */ - public void addArtistId(Document document) { - StringFieldValue artistString = (StringFieldValue) document.getFieldValue(artistField); - HttpURLConnection conn = null; - try { - String url = spotifyUrl + java.net.URLEncoder.encode(artistString.getString(), "UTF-8"); - conn = getConnection(url); - String artistId = parseSpotifyResponse(new InputStreamReader(conn.getInputStream(), "UTF-8")); - if (artistId == null) { - return; - } - document.setFieldValue(artistIdField, new StringFieldValue(artistId)); - - } catch (Exception e) { - log.warning("Error: " ); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - } - - /** - * @param streamReader the response to read from - * @return artist id from spotify or null if not found - * @throws IOException - */ - - private String parseSpotifyResponse(InputStreamReader streamReader) throws IOException { - // Read JSON data from API - BufferedReader reader = new BufferedReader(streamReader); - StringBuilder builder = new StringBuilder(); - for (String line; (line = reader.readLine()) != null; ) { - builder.append(line).append("\n"); - } - - // Parse the JSON to find the first artist item returned or null if not found - try { - JSONObject json = new JSONObject(builder.toString()); - JSONObject artists = json.getJSONObject("artists"); - JSONArray items = artists.getJSONArray("items"); - JSONObject artist = items.getJSONObject(0); - return artist.getString("id"); - } catch (JSONException e) { - return null; - } - - } - - /** - * Establishes an HTTP Connection - * - * @param inputUrl a String giving the URL to connect to - * @return an HttpURLConnection - * @throws java.io.IOException when connection to inputUrl failed - */ - private HttpURLConnection getConnection(String inputUrl) throws IOException { - URL url = new URL(inputUrl); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - - conn.setRequestProperty("User-Agent", "Vespa Tutorial DocProc"); - conn.setReadTimeout(10000); - conn.setConnectTimeout(5000); - conn.connect(); - - return conn; - } -} - diff --git a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java deleted file mode 100644 index cf4c1fc8e34..00000000000 --- a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; - -/** - * A searcher adding a new hit. - * - * @author Joe Developer - */ -public class ExampleSearcher extends Searcher { - public static final String hitId = "ExampleHit"; - private final String message; - - public ExampleSearcher(MessageConfig config) { - message = config.message(); - } - - public Result search(Query query, Execution execution) { - Hit hit = new Hit(hitId); - hit.setField("message", message); - - Result result = execution.search(query); - result.hits().add(hit); - return result; - } -} diff --git a/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def b/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def deleted file mode 100644 index d2a2bc5407a..00000000000 --- a/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def +++ /dev/null @@ -1,3 +0,0 @@ -version=1 -namespace=example -message string default="Hello, World!"
\ No newline at end of file diff --git a/node-admin/basic-search-for-docker/src/test/application/services.xml b/node-admin/basic-search-for-docker/src/test/application/services.xml deleted file mode 100644 index 388ab448f9c..00000000000 --- a/node-admin/basic-search-for-docker/src/test/application/services.xml +++ /dev/null @@ -1,20 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<services version='1.0'> - <admin version='3.0'> - <nodes count="1" /> - </admin> - - <!-- duplication of src/main/application due to Ticket 6904654 --> - <jdisc version='1.0' id='default'> - <search> - <chain id="default" inherits="vespa"> - <searcher id="com.yahoo.example.ExampleSearcher" bundle="basic-application"> - <config name="example.message"> - <message>Hello, Vespa!</message> - </config> - </searcher> - </chain> - </search> - <nodes count="1" /> - </jdisc> -</services> diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java deleted file mode 100644 index 4a6c48073c1..00000000000 --- a/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java +++ /dev/null @@ -1,16 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.base.StagingAutoTestConfigBuilder; -import com.yahoo.vespa.tenant.systemtest.base.StagingTest; -import org.junit.Test; - -import java.io.IOException; - -/** - * This test is only run if there is a production instance to get queries and documents from, otherwise the tests - * are marked as ignored automatically. - */ -public class BasicSearchConditionalStagingTest extends StagingTest { - @Test - public void testProdQueriesAndDocuments() throws IOException { - testAutomatically(new StagingAutoTestConfigBuilder().withRoute("music-index").withDocumentType("music").build()); - } -} diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java deleted file mode 100644 index 89bf52a9870..00000000000 --- a/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java +++ /dev/null @@ -1,27 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.base.StagingTest; -import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; -import com.yahoo.vespa.tenant.systemtest.blackbox.Report; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -public class BasicSearchStagingTest extends StagingTest { - @Test - public void testSearchAndFeeding() throws Exception { - - /** - * Generates 100 documents and test recall while feeding. - */ - Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( - " {\n" + - " \"put\": \"id:sampleapp:music::::$1\",\n" + - " \"fields\": {\n" + - " \"title\": \"$2\"\n" + - " }\n" + - " }") - .numberOfDocumentsToFeed(100).build(getStagingTestInstance().getDefaultVespaEndpoint().getUri()) - .testFeedingAndRecall(); - assertThat(report.allSuccess(), is(true)); - } -} diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java deleted file mode 100644 index e9e72ca4d8c..00000000000 --- a/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java +++ /dev/null @@ -1,50 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.Endpoint; -import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; -import com.yahoo.vespa.tenant.systemtest.base.SystemTest; -import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; -import com.yahoo.vespa.tenant.systemtest.blackbox.Report; -import org.junit.Before; -import org.junit.Test; - -import java.net.URI; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * Tests running in CI-pipeline. In a regular vespa system this will be triggered whenever a commit is made and - * run before code is pushed to production. - * - * @author dybdahl - */ -public class BasicSearchSystemTest extends SystemTest { - @Override - /** - * In order to develop and test the system tests, you can create a dev instance and push this - * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this - * function to point to this instance. This is only for testing and debugging the test. - */ - protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { - return new VespaEndpoints.Builder().fromPom().inDevCluster() - .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); - } - - - @Test - public void testSearchAndFeeding() throws Exception { - - /** - * Generates 100 documents and test recall while feeding. - */ - Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( - " {\n" + - " \"put\": \"id:sampleapp:music::::$1\",\n" + - " \"fields\": {\n" + - " \"title\": \"$2\"\n" + - " }\n" + - " }") - .numberOfDocumentsToFeed(100).build(getSystemTestsInstance().getDefaultVespaEndpoint().getUri()) - .testFeedingAndRecall(); - assertThat(report.allSuccess(), is(true)); - } -} diff --git a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java deleted file mode 100644 index 87592f2a2ba..00000000000 --- a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.application.Application; -import com.yahoo.application.Networking; -import com.yahoo.application.container.Search; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author tonyv - */ -public class ExampleSearcherTest { - @Test - public void hit_is_added() throws Exception { - try (Application app = Application.fromApplicationPackage( - Paths.get("src/test/application"), - Networking.disable)) - { - Search search = app.getJDisc("jdisc").search(); - Result result = search.process(ComponentSpecification.fromString("default"), new Query("?query=ignored")); - - Hit hit = result.hits().get(ExampleSearcher.hitId); - assertNotNull("Hit was not added by ExampleSearcher", hit); - - Object messageFromConfig = "Hello, Vespa!"; - assertThat(hit.getField("message"), is(messageFromConfig)); - } - } -} diff --git a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java deleted file mode 100644 index 2bed440868e..00000000000 --- a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.vespa.tenant.systemtest.Feed; -import com.yahoo.vespa.tenant.systemtest.FeedResult; -import com.yahoo.vespa.tenant.systemtest.Query; -import com.yahoo.vespa.tenant.systemtest.QueryResult; -import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; -import com.yahoo.vespa.tenant.systemtest.base.MutableVespaEndpoint; -import com.yahoo.vespa.tenant.systemtest.base.SystemTest; -import com.yahoo.vespa.tenant.systemtest.hitchecker.HitChecker; -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class ExampleSystemTest extends SystemTest { - @Override - /** - * In order to develop and test the system tests, you can create a dev instance and push this - * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this - * function to point to this instance. This is only for testing and debugging the test. - */ - protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { - return new VespaEndpoints.Builder().fromPom().inDevCluster() - .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); - } - @Test - public void testWithOneDocument() throws Exception { - MutableVespaEndpoint endpoint = getSystemTestsInstance().getDefaultVespaEndpoint(); - FeedResult feedResult = endpoint.feed(Feed.createFromResource("/minifeed.json")); - assertThat(feedResult.numOk(), is(1l)); - QueryResult result = endpoint.search(new Query("bad")); - assertThat(result.totalHitCount(), is(1l)); - result.expectHit(1, new HitChecker() - .fieldRegex("title", ".*Bad") - .relevance(0.254, 0.2) - .fieldNull("nosuchfield")); - } -} diff --git a/node-admin/basic-search-for-docker/src/test/resources/minifeed.json b/node-admin/basic-search-for-docker/src/test/resources/minifeed.json deleted file mode 100644 index 2fd5d9e2053..00000000000 --- a/node-admin/basic-search-for-docker/src/test/resources/minifeed.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "fields": { - "album": "Bad", - "artist": "Michael Jackson", - "title": "Bad", - "year": 1987 - }, - "put": "id:tutorial:music::1" - } -] - diff --git a/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch b/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch deleted file mode 100644 index b4e0745fcc2..00000000000 --- a/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/main/application/searchdefinitions/music.sd b/src/main/application/searchdefinitions/music.sd -index d6bcaac..158c217 100644 ---- a/src/main/application/searchdefinitions/music.sd -+++ b/src/main/application/searchdefinitions/music.sd -@@ -16,7 +16,7 @@ search music { - } - - field duration type int { -- indexing: summary -+ indexing: summary | attribute - } - - field year type int { diff --git a/node-admin/include/http-server.xml b/node-admin/configserver-app/hosted-vespa/http-server.xml index f77d392de97..f77d392de97 100644 --- a/node-admin/include/http-server.xml +++ b/node-admin/configserver-app/hosted-vespa/http-server.xml diff --git a/node-admin/include/node-flavors.xml b/node-admin/configserver-app/node-flavors.xml index 7d22975994a..7d22975994a 100644 --- a/node-admin/include/node-flavors.xml +++ b/node-admin/configserver-app/node-flavors.xml diff --git a/node-admin/include/start-config-server.sh b/node-admin/configserver-app/start-config-server.sh index a592ec3cbd3..a592ec3cbd3 100755 --- a/node-admin/include/start-config-server.sh +++ b/node-admin/configserver-app/start-config-server.sh diff --git a/node-admin/include/Dockerfile.template b/node-admin/include/Dockerfile.template deleted file mode 100644 index 53f8bfac18f..00000000000 --- a/node-admin/include/Dockerfile.template +++ /dev/null @@ -1,33 +0,0 @@ -# NOTE: Whenever the vespa-base version changes, review pom.xml. A fixed version of vespa-base contains components -# built from a certain vespa revision, while the docker image you will build with this Dockerfile contains components -# (i.e. the node-admin jar file) built from HEAD. Since dependencies change over time, there may be mismatches. -# Therefore, one of the following may be necessary for one or more dependencies in pom.xml: -# a) Hard-coding a version override. -# b) Changing a hard-coded version override. -# c) Removing a hard-coded version override. -# -# Because of the way docker creates and caches docker image layers, the command should be in order of likelihood -# of change (Files that are updated most often should be at the bottom). -# -# See build.sh for the meaning of $NODE_ADMIN_FROM_IMAGE. -FROM $NODE_ADMIN_FROM_IMAGE - -# Be aware that this Dockerfile is not being used in any pipelines and has no relation to production environments etc. -# It is here for developers' convenience - it allows building and experimenting with node-admin locally. - -# Things for convenience. -RUN yum install -y tcpdump -ADD root-bashrc /root/.bashrc - -# Make config-server aware of node flavor 'docker'. -ADD node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml - -# Make config-server also listen to 4080 -ADD http-server.xml $VESPA_HOME/conf/configserver-app/hosted-vespa/http-server.xml - -# Entrypoint for running config server in a container. -ADD start-config-server.sh $VESPA_HOME/bin/start-config-server.sh - -# Included in base image, but here overridden with local modifications. -# TODO: Update the source instead. -ADD start-services.sh $VESPA_HOME/bin/start-services.sh diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh index ed5f42fe5e1..b3d3dbea440 100755..100644 --- a/node-admin/include/nodectl-instance.sh +++ b/node-admin/include/nodectl-instance.sh @@ -22,6 +22,10 @@ # BEGIN environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +function debug { + echo "$(date -Is) $*" +} + findpath () { myname=${0} mypath=${myname%/*} @@ -109,34 +113,37 @@ Resume() { # Start all services, can be seen as a reboot of a non-Docker node Start() { - echo "Configuring rsyslog service to work" + debug "Configuring rsyslog service to work" # Disable kernel log module sed -i.bak 's/^\$ModLoad imklog/#$ModLoad imklog/' /etc/rsyslog.conf - echo "Starting rsyslog service" + debug "Starting rsyslog service" service rsyslog start - echo "Starting crond service" + debug "Starting crond service" service crond start - echo "Yinst settings" - yinst set zpe_policy_updater.domains=$ATHENS_DOMAIN - yinst set zpe_policy_updater.autostart=on - echo "Starting all yinst packages" + debug "Yinst settings" + yinst set zpe_policy_updater.domains=$ATHENS_DOMAIN \ + zpe_policy_updater.autostart=on \ + zpe_policy_updater.cron_start_hour=* \ + zpe_policy_updater.cron_start_min=*/30 \ + zpe_policy_updater.cron_start_delay=30 + debug "Starting all yinst packages" # Start yinst the way it is done when a non-Docker node is booted. # As this is implemented in yinst now (2017-02-08), this will take care of # cleaning up /home/y/tmp and /home/y/var/run /etc/rc.d/init.d/yinst start - echo "yinst started, exited with $?" + debug "yinst started, exited with $?" } # Stop all services, can be seen as a shutdown of a non-Docker node Stop() { - echo "Stopping services and other yinst packages running" + debug "Stopping services and other yinst packages running" # Stop yinst the way it is done when a non-Docker node is shutdown. /etc/rc.d/init.d/yinst stop - echo "Stopping crond service" + debug "Stopping crond service" service crond stop - echo "Stopping rsyslog service" + debug "Stopping rsyslog service" service rsyslog stop } @@ -181,4 +188,4 @@ main() { fi } -main "$@" +main "$@"
\ No newline at end of file diff --git a/node-admin/include/root-bashrc b/node-admin/include/root-bashrc deleted file mode 100644 index fda1752c85f..00000000000 --- a/node-admin/include/root-bashrc +++ /dev/null @@ -1 +0,0 @@ -export TERM=xterm diff --git a/node-admin/include/start-services.sh b/node-admin/include/start-services.sh deleted file mode 100755 index 0fe5c1c0724..00000000000 --- a/node-admin/include/start-services.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# TODO: This file duplicates a file in the base image (with local modifications). -# Update the file there. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - # ensure it ends with "/" : - VESPA_HOME=${VESPA_HOME%/}/ - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findroot - -# END environment bootstrap section - -export LC_ALL=C - -function wait_for_network_up { - while true - do - for config_server_host in $(echo $CONFIG_SERVER_ADDRESS | tr "," " ") - do - ping -c 1 -W 3 $config_server_host && return - sleep 1 - done - done -} - -if [ -z $CONFIG_SERVER_ADDRESS ] -then - echo "CONFIG_SERVER_ADDRESS must be set." - exit -1 -fi - -chown yahoo $VESPA_HOME/var/jdisc_container - -# Local modification; fixes ownership issues for vespa node running in container. -chown yahoo $VESPA_HOME/var/zookeeper - -if [ -d "$VESPA_HOME/logs" ] -then - chmod 1777 $VESPA_HOME/logs -fi - -yinst set services.addr_configserver=$CONFIG_SERVER_ADDRESS -wait_for_network_up -yinst start services -logfmt -n -f diff --git a/node-admin/node-admin-zone-app/services.xml b/node-admin/node-admin-zone-app/services.xml new file mode 100644 index 00000000000..d8740e5bbdb --- /dev/null +++ b/node-admin/node-admin-zone-app/services.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8" ?> +<services version="1.0" application-type="hosted-infrastructure" xmlns:deploy="vespa" xmlns:preprocess="properties"> + + <jdisc id="node-admin" jetty="true" version="1.0"> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> + <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> + <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> + + <config name='vespa.hosted.dockerapi.docker'> + <isRunningLocally>true</isRunningLocally> + </config> + + <nodes jvmargs="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5555 -Dvespa.freezedetector.disable=true" type="host"/> + </jdisc> + +</services> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java index 5ad60475276..bc5107b7db0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java @@ -14,9 +14,11 @@ import java.util.Optional; public class ContainerNodeSpec { public final String hostname; public final Optional<DockerImage> wantedDockerImage; + public final Optional<DockerImage> currentDockerImage; public final Node.State nodeState; public final String nodeType; public final String nodeFlavor; + public final Optional<String> wantedVespaVersion; public final Optional<String> vespaVersion; public final Optional<Owner> owner; public final Optional<Membership> membership; @@ -31,9 +33,11 @@ public class ContainerNodeSpec { public ContainerNodeSpec( final String hostname, final Optional<DockerImage> wantedDockerImage, + final Optional<DockerImage> currentDockerImage, final Node.State nodeState, final String nodeType, final String nodeFlavor, + final Optional<String> wantedVespaVersion, final Optional<String> vespaVersion, final Optional<Owner> owner, final Optional<Membership> membership, @@ -51,9 +55,11 @@ public class ContainerNodeSpec { this.hostname = hostname; this.wantedDockerImage = wantedDockerImage; + this.currentDockerImage = currentDockerImage; this.nodeState = nodeState; this.nodeType = nodeType; this.nodeFlavor = nodeFlavor; + this.wantedVespaVersion = wantedVespaVersion; this.vespaVersion = vespaVersion; this.owner = owner; this.membership = membership; @@ -75,9 +81,11 @@ public class ContainerNodeSpec { return Objects.equals(hostname, that.hostname) && Objects.equals(wantedDockerImage, that.wantedDockerImage) && + Objects.equals(currentDockerImage, that.currentDockerImage) && Objects.equals(nodeState, that.nodeState) && Objects.equals(nodeType, that.nodeType) && Objects.equals(nodeFlavor, that.nodeFlavor) && + Objects.equals(wantedVespaVersion, that.wantedVespaVersion) && Objects.equals(vespaVersion, that.vespaVersion) && Objects.equals(owner, that.owner) && Objects.equals(membership, that.membership) && @@ -95,9 +103,11 @@ public class ContainerNodeSpec { return Objects.hash( hostname, wantedDockerImage, + currentDockerImage, nodeState, nodeType, nodeFlavor, + wantedVespaVersion, vespaVersion, owner, membership, @@ -115,9 +125,11 @@ public class ContainerNodeSpec { return getClass().getSimpleName() + " {" + " hostname=" + hostname + " wantedDockerImage=" + wantedDockerImage + + " currentDockerImage=" + currentDockerImage + " nodeState=" + nodeState + " nodeType = " + nodeType + " nodeFlavor = " + nodeFlavor + + " wantedVespaVersion = " + wantedVespaVersion + " vespaVersion = " + vespaVersion + " owner = " + owner + " membership = " + membership @@ -227,9 +239,11 @@ public class ContainerNodeSpec { public static class Builder { private String hostname; private Optional<DockerImage> wantedDockerImage = Optional.empty(); + private Optional<DockerImage> currentDockerImage = Optional.empty(); private Node.State nodeState; private String nodeType; private String nodeFlavor; + private Optional<String> wantedVespaVersion = Optional.empty(); private Optional<String> vespaVersion = Optional.empty(); private Optional<Owner> owner = Optional.empty(); private Optional<Membership> membership = Optional.empty(); @@ -250,6 +264,8 @@ public class ContainerNodeSpec { nodeFlavor(nodeSpec.nodeFlavor); nodeSpec.wantedDockerImage.ifPresent(this::wantedDockerImage); + nodeSpec.currentDockerImage.ifPresent(this::currentDockerImage); + nodeSpec.wantedVespaVersion.ifPresent(this::wantedVespaVersion); nodeSpec.vespaVersion.ifPresent(this::vespaVersion); nodeSpec.owner.ifPresent(this::owner); nodeSpec.membership.ifPresent(this::membership); @@ -272,6 +288,11 @@ public class ContainerNodeSpec { return this; } + public Builder currentDockerImage(DockerImage currentDockerImage) { + this.currentDockerImage = Optional.of(currentDockerImage); + return this; + } + public Builder nodeState(Node.State nodeState) { this.nodeState = nodeState; return this; @@ -286,6 +307,11 @@ public class ContainerNodeSpec { return this; } + public Builder wantedVespaVersion(String wantedVespaVersion) { + this.wantedVespaVersion = Optional.of(wantedVespaVersion); + return this; + } + public Builder vespaVersion(String vespaVersion) { this.vespaVersion = Optional.of(vespaVersion); return this; @@ -337,8 +363,8 @@ public class ContainerNodeSpec { } public ContainerNodeSpec build() { - return new ContainerNodeSpec(hostname, wantedDockerImage, nodeState, nodeType, nodeFlavor, - vespaVersion, owner, membership, + return new ContainerNodeSpec(hostname, wantedDockerImage, currentDockerImage, nodeState, nodeType, nodeFlavor, + wantedVespaVersion, vespaVersion, owner, membership, wantedRestartGeneration, currentRestartGeneration, wantedRebootGeneration, currentRebootGeneration, minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index aa7285ec17c..a012de36b62 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -12,7 +12,6 @@ import java.util.List; import java.util.Optional; public interface DockerOperations { - Optional<String> getVespaVersion(ContainerName containerName); void startContainer(ContainerName containerName, ContainerNodeSpec nodeSpec); @@ -23,7 +22,7 @@ public interface DockerOperations { Optional<Container> getContainer(ContainerName containerName); - void scheduleDownloadOfImage(ContainerName containerName, ContainerNodeSpec nodeSpec, Runnable callback); + void scheduleDownloadOfImage(ContainerName containerName, DockerImage dockerImage, Runnable callback); ProcessResult executeCommandInContainerAsRoot(ContainerName containerName, String... command); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index fafbf3e2563..c2855da85a0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -1,38 +1,31 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import com.google.common.base.Joiner; -import com.google.common.io.CharStreams; -import com.yahoo.net.HostName; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.DockerImpl; import com.yahoo.vespa.hosted.dockerapi.DockerNetworkCreator; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.io.IOException; -import java.io.InputStreamReader; import java.net.Inet6Address; import java.net.InetAddress; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -43,7 +36,6 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; */ public class DockerOperationsImpl implements DockerOperations { public static final String NODE_PROGRAM = Defaults.getDefaults().underVespaHome("bin/vespa-nodectl"); - private static final String[] GET_VESPA_VERSION_COMMAND = new String[]{NODE_PROGRAM, "vespa-version"}; private static final String[] RESUME_NODE_COMMAND = new String[]{NODE_PROGRAM, "resume"}; private static final String[] SUSPEND_NODE_COMMAND = new String[]{NODE_PROGRAM, "suspend"}; @@ -93,39 +85,10 @@ public class DockerOperationsImpl implements DockerOperations { private final Docker docker; private final Environment environment; - private final Consumer<List<String>> commandExecutor; - private GaugeWrapper numberOfRunningContainersGauge; - public DockerOperationsImpl(Docker docker, Environment environment, MetricReceiverWrapper metricReceiver) { - this(docker, environment, metricReceiver, DockerOperationsImpl::runCommand); - } - - DockerOperationsImpl(Docker docker, Environment environment, MetricReceiverWrapper metricReceiver, - Consumer<List<String>> commandExecutor) { + public DockerOperationsImpl(Docker docker, Environment environment) { this.docker = docker; this.environment = environment; - setMetrics(metricReceiver); - this.commandExecutor = commandExecutor; - } - - @Override - public Optional<String> getVespaVersion(ContainerName containerName) { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - - ProcessResult result = docker.executeInContainer(containerName, DockerOperationsImpl.GET_VESPA_VERSION_COMMAND); - if (!result.isSuccess()) { - logger.warning("Container " + containerName.asString() + ": Command " - + Arrays.toString(DockerOperationsImpl.GET_VESPA_VERSION_COMMAND) + " failed: " + result); - return Optional.empty(); - } - Optional<String> vespaVersion = parseVespaVersion(result.getOutput()); - if (vespaVersion.isPresent()) { - return vespaVersion; - } else { - logger.warning("Container " + containerName.asString() + ": Failed to parse vespa version from " - + result.getOutput()); - return Optional.empty(); - } } // Returns empty if vespa version cannot be parsed. @@ -160,6 +123,10 @@ public class DockerOperationsImpl implements DockerOperations { .withUlimit("core", -1, -1) .withAddCapability("SYS_PTRACE"); // Needed for gcore, pstack etc. + if (environment.isRunningLocally()) { + command.withEntrypoint("/usr/local/bin/start-services.sh", "--run-local"); + } + command.withVolume("/etc/hosts", "/etc/hosts"); for (String pathInNode : DIRECTORIES_TO_MOUNT.keySet()) { String pathInHost = environment.pathInHostFromPathInNode(containerName, pathInNode).toString(); @@ -200,8 +167,6 @@ public class DockerOperationsImpl implements DockerOperations { } catch (IOException e) { throw new RuntimeException("Failed to create container " + containerName.asString(), e); } - - numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); } @Override @@ -215,7 +180,6 @@ public class DockerOperationsImpl implements DockerOperations { logger.info("Deleting container " + containerName.asString()); docker.deleteContainer(containerName); - numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); } // Returns true if scheduling download @@ -256,22 +220,21 @@ public class DockerOperationsImpl implements DockerOperations { */ private void setupContainerNetworkingWithScript(ContainerName containerName) throws IOException { InetAddress hostDefaultGateway = DockerNetworkCreator.getDefaultGatewayLinux(true); - executeCommandInNetworkNamespace(containerName, new String[]{ - "route", "-A", "inet6", "add", "default", "gw", hostDefaultGateway.getHostAddress(), "dev", "eth1"}); + executeCommandInNetworkNamespace(containerName, + "route", "-A", "inet6", "add", "default", "gw", hostDefaultGateway.getHostAddress(), "dev", "eth1"); } @Override - public void scheduleDownloadOfImage(ContainerName containerName, final ContainerNodeSpec nodeSpec, Runnable callback) { + public void scheduleDownloadOfImage(ContainerName containerName, DockerImage dockerImage, Runnable callback) { PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - logger.info("Schedule async download of " + nodeSpec.wantedDockerImage.get()); - final CompletableFuture<DockerImage> asyncPullResult = docker.pullImageAsync(nodeSpec.wantedDockerImage.get()); - asyncPullResult.whenComplete((dockerImage, throwable) -> { + logger.info("Schedule async download of " + dockerImage); + final CompletableFuture<DockerImage> asyncPullResult = docker.pullImageAsync(dockerImage); + asyncPullResult.whenComplete((image, throwable) -> { if (throwable != null) { - logger.warning("Failed to pull " + nodeSpec.wantedDockerImage, throwable); + logger.warning("Failed to pull " + dockerImage, throwable); return; } - assert nodeSpec.wantedDockerImage.get().equals(dockerImage); callback.run(); }); } @@ -305,20 +268,23 @@ public class DockerOperationsImpl implements DockerOperations { .orElseThrow(() -> new RuntimeException("PID not found for container with name: " + containerName.asString())); - final List<String> wrappedCommand = new LinkedList<>(); - wrappedCommand.add("sudo"); - wrappedCommand.add("-n"); // Run non-interactively and fail if a password is required - wrappedCommand.add("nsenter"); - wrappedCommand.add(String.format("--net=/host/proc/%d/ns/net", containerPid)); - wrappedCommand.add("--"); - wrappedCommand.addAll(Arrays.asList(command)); + final String[] wrappedCommand = Stream.concat( + Stream.of("nsenter", String.format("--net=/host/proc/%d/ns/net", containerPid), "--"), + Stream.of(command)) + .toArray(String[]::new); try { - commandExecutor.accept(wrappedCommand); - } catch (Exception e) { - logger.error(String.format("Failed to execute %s in network namespace for %s (PID = %d)", - Arrays.toString(command), containerName.asString(), containerPid)); - throw new RuntimeException(e); + ProcessResult result = docker.executeInContainerAsRoot(new ContainerName("node-admin"), 60L, wrappedCommand); + if (! result.isSuccess()) { + String msg = String.format("Failed to execute %s in network namespace for %s (PID = %d)", + Arrays.toString(wrappedCommand), containerName.asString(), containerPid); + logger.error(msg); + throw new RuntimeException(msg); + } + } catch (DockerExecTimeoutException e) { + logger.warning(String.format("Timed out while executing %s in network namespace for %s (PID = %d)", + Arrays.toString(wrappedCommand), containerName.asString(), containerPid)); + throw e; } } @@ -351,30 +317,4 @@ public class DockerOperationsImpl implements DockerOperations { public void deleteUnusedDockerImages() { docker.deleteUnusedDockerImages(); } - - private void setMetrics(MetricReceiverWrapper metricReceiver) { - Dimensions dimensions = new Dimensions.Builder() - .add("host", HostName.getLocalhost()) - .add("role", "docker").build(); - - numberOfRunningContainersGauge = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "containers.running"); - - // Some containers could already be running, count them and initialize to that value - numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); - } - - private static void runCommand(List<String> command) { - try { - final Process process = new ProcessBuilder(command) - .redirectErrorStream(true) - .start(); - final String output = CharStreams.toString(new InputStreamReader(process.getInputStream())); - final int resultCode = process.waitFor(); - if (resultCode != 0) { - throw new RuntimeException("Command " + Joiner.on(' ').join(command) + " failed: " + output); - } - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 99a07bcda60..4df6ab52325 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -15,13 +15,17 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; +import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -49,13 +53,15 @@ public class StorageMaintainer { private final CounterWrapper numberOfNodeAdminMaintenanceFails; private final Docker docker; private final Environment environment; + private final Clock clock; - private Map<ContainerName, MetricsCache> metricsCacheByContainerName = new ConcurrentHashMap<>(); + private Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>(); - public StorageMaintainer(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) { + public StorageMaintainer(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { this.docker = docker; this.environment = environment; + this.clock = clock; Dimensions dimensions = new Dimensions.Builder() .add("host", HostName.getLocalhost()) @@ -64,31 +70,76 @@ public class StorageMaintainer { numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails"); } - public Map<String, Number> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) { + public void writeMetricsConfig(ContainerName containerName, ContainerNodeSpec nodeSpec) { + final Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, "/etc/yamas-agent/"); + + Path vespaCheckPath = Paths.get(getDefaults().underVespaHome("libexec/yms/yms_check_vespa")); + SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("vespa", 60, vespaCheckPath, "all") + .withTag("namespace", "Vespa") + .withTag("role", "tenants") + .withTag("flavor", nodeSpec.nodeFlavor) + .withTag("state", nodeSpec.nodeState.toString()) + .withTag("zone", environment.getZone()) + .withTag("parentHostname", environment.getParentHostHostname()); + + nodeSpec.owner.ifPresent(owner -> + scheduleMaker + .withTag("tenantName", owner.tenant) + .withTag("app", owner.application + "." + owner.instance)); + + nodeSpec.membership.ifPresent(membership -> + scheduleMaker + .withTag("clustertype", membership.clusterType) + .withTag("clusterid", membership.clusterId)); + + nodeSpec.vespaVersion.ifPresent(version -> scheduleMaker.withTag("vespaVersion", version)); + + try { + scheduleMaker.writeTo(yamasAgentFolder); + final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; + docker.executeInContainerAsRoot(containerName, restartYamasAgent); + } catch (IOException e) { + throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); + } + } + + public void writeFilebeatConfig(ContainerName containerName, ContainerNodeSpec nodeSpec) { + PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); + try { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); + Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec); + if (!config.isPresent()) { + logger.error("Was not able to generate a config for filebeat, ignoring filebeat file creation." + nodeSpec.toString()); + return; + } + Path filebeatPath = environment.pathInNodeAdminFromPathInNode(containerName, "/etc/filebeat/filebeat.yml"); + Files.write(filebeatPath, config.get().getBytes()); + logger.info("Wrote filebeat config."); + } catch (Throwable t) { + logger.error("Failed writing filebeat config; " + nodeSpec, t); + } + } + + public Optional<Long> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) { // Calculating disk usage is IO expensive operation and its value changes relatively slowly, we want to perform // that calculation rarely. Additionally, we spread out the calculation for different containers by adding // a random deviation. - if (! metricsCacheByContainerName.containsKey(containerName) || - metricsCacheByContainerName.get(containerName).nextUpdateAt.isBefore(Instant.now())) { - long distributedSecs = (long) (intervalSec * (0.5 + Math.random())); - MetricsCache metricsCache = new MetricsCache(Instant.now().plusSeconds(distributedSecs)); - + MaintenanceThrottler maintenanceThrottler = getMaintenanceThrottlerFor(containerName); + if (maintenanceThrottler.shouldUpdateDiskUsageNow()) { // Throttle to one disk usage calculation at a time. synchronized (monitor) { PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); Path containerDir = environment.pathInNodeAdminFromPathInNode(containerName, "/home/"); try { long used = getDiscUsedInBytes(containerDir); - metricsCache.metrics.put("node.disk.used", used); + maintenanceThrottler.setDiskUsage(used); } catch (Throwable e) { logger.error("Problems during disk usage calculations: " + e.getMessage()); } } - - metricsCacheByContainerName.put(containerName, metricsCache); } - return metricsCacheByContainerName.get(containerName).metrics; + return maintenanceThrottler.diskUsage; } // Public for testing @@ -121,6 +172,8 @@ public class StorageMaintainer { * Deletes old log files for vespa, nginx, logstash, etc. */ public void removeOldFilesFromNode(ContainerName containerName) { + if (! getMaintenanceThrottlerFor(containerName).shouldRemoveOldFilesNow()) return; + MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); String[] pathsToClean = {"/home/y/logs/elasticsearch2", "/home/y/logs/logstash2", "/home/y/logs/daemontools_y", "/home/y/logs/nginx", "/home/y/logs/vespa"}; @@ -152,15 +205,18 @@ public class StorageMaintainer { maintainerExecutor.addJob("delete-files") .withArgument("basePath", fileDistrDir) .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) - .withArgument("recursive", false); + .withArgument("recursive", true); maintainerExecutor.execute(); + getMaintenanceThrottlerFor(containerName).updateNextRemoveOldFilesTime(); } /** * Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(ContainerName containerName, ContainerNodeSpec nodeSpec, Environment environment) { + if (! getMaintenanceThrottlerFor(containerName).shouldHandleCoredumpsNow()) return; + Map<String, Object> attributes = new HashMap<>(); attributes.put("hostname", nodeSpec.hostname); attributes.put("parent_hostname", HostName.getLocalhost()); @@ -186,27 +242,40 @@ public class StorageMaintainer { .withArgument("doneCoredumpsPath", environment.pathInNodeAdminToDoneCoredumps()) .withArgument("coredumpsPath", environment.pathInNodeAdminFromPathInNode(containerName, "/home/y/var/crash")) .withArgument("attributes", attributes); + maintainerExecutor.execute(); + getMaintenanceThrottlerFor(containerName).updateNextHandleCoredumpsTime(); } /** * Deletes old * * archived app data - * * JDisc logs + * * Vespa logs + * * Filedistribution files */ public void cleanNodeAdmin() { + if (! getMaintenanceThrottlerFor(NODE_ADMIN).shouldRemoveOldFilesNow()) return; + MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); maintainerExecutor.addJob("delete-directories") .withArgument("basePath", environment.getPathResolver().getApplicationStoragePathForNodeAdmin()) .withArgument("maxAgeSeconds", Duration.ofDays(7).getSeconds()) .withArgument("dirNameRegex", "^" + Pattern.quote(Environment.APPLICATION_STORAGE_CLEANUP_PATH_PREFIX)); - Path nodeAdminJDiskLogsPath = environment.pathInNodeAdminFromPathInNode(NODE_ADMIN, "/home/y/logs/jdisc_core/"); + Path nodeAdminJDiskLogsPath = environment.pathInNodeAdminFromPathInNode(NODE_ADMIN, "/home/y/logs/vespa/"); maintainerExecutor.addJob("delete-files") .withArgument("basePath", nodeAdminJDiskLogsPath) .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) .withArgument("recursive", false); + + Path fileDistrDir = environment.pathInNodeAdminFromPathInNode(NODE_ADMIN, "/home/y/var/db/vespa/filedistribution"); + maintainerExecutor.addJob("delete-files") + .withArgument("basePath", fileDistrDir) + .withArgument("maxAgeSeconds", Duration.ofDays(31).getSeconds()) + .withArgument("recursive", true); + maintainerExecutor.execute(); + getMaintenanceThrottlerFor(NODE_ADMIN).updateNextRemoveOldFilesTime(); } /** @@ -222,6 +291,7 @@ public class StorageMaintainer { .withArgument("to", environment.pathInNodeAdminToNodeCleanup(containerName)); maintainerExecutor.execute(); + getMaintenanceThrottlerFor(containerName).reset(); } @@ -260,11 +330,7 @@ public class StorageMaintainer { return job; } - ProcessResult execute() { - String classPath = String.join(":", - "/home/y/lib/jars/node-maintainer-jar-with-dependencies.jar", - "/home/y/lib/jars/vespajlib.jar"); - + void execute() { String args; try { args = objectMapper.writeValueAsString(jobs); @@ -272,7 +338,8 @@ public class StorageMaintainer { throw new RuntimeException("Failed transform list of maintenance jobs to JSON"); } - String[] command = {"java", "-cp", classPath, + String[] command = {"java", + "-cp", "/home/y/lib/jars/node-maintainer-jar-with-dependencies.jar", "-Dvespa.log.target=file:" + getDefaults().underVespaHome("logs/vespa/maintainer.log"), "com.yahoo.vespa.hosted.node.maintainer.Maintainer", args}; ProcessResult result = docker.executeInContainerAsRoot(executeIn, command); @@ -281,7 +348,6 @@ public class StorageMaintainer { numberOfNodeAdminMaintenanceFails.add(); throw new RuntimeException("Failed to run maintenance jobs: " + args + result); } - return result; } } @@ -303,12 +369,54 @@ public class StorageMaintainer { } } - private static class MetricsCache { - private final Instant nextUpdateAt; - private final Map<String, Number> metrics = new HashMap<>(); + private MaintenanceThrottler getMaintenanceThrottlerFor(ContainerName containerName) { + if (! maintenanceThrottlerByContainerName.containsKey(containerName)) { + maintenanceThrottlerByContainerName.put(containerName, new MaintenanceThrottler()); + } + + return maintenanceThrottlerByContainerName.get(containerName); + } + + private class MaintenanceThrottler { + private Instant nextDiskUsageUpdateAt; + private Instant nextRemoveOldFilesAt; + private Instant nextHandleOldCoredumpsAt; + private Optional<Long> diskUsage = Optional.empty(); + + MaintenanceThrottler() { + reset(); + } + + boolean shouldUpdateDiskUsageNow() { + return !nextDiskUsageUpdateAt.isAfter(clock.instant()); + } + + void setDiskUsage(long diskUsage) { + this.diskUsage = Optional.of(diskUsage); + long distributedSecs = (long) (intervalSec * (0.5 + Math.random())); + nextDiskUsageUpdateAt = clock.instant().plusSeconds(distributedSecs); + } + + void updateNextRemoveOldFilesTime() { + nextRemoveOldFilesAt = clock.instant().plus(Duration.ofHours(1)); + } + + boolean shouldRemoveOldFilesNow() { + return !nextRemoveOldFilesAt.isAfter(clock.instant()); + } + + void updateNextHandleCoredumpsTime() { + nextHandleOldCoredumpsAt = clock.instant().plus(Duration.ofHours(1)); + } + + boolean shouldHandleCoredumpsNow() { + return !nextHandleOldCoredumpsAt.isAfter(clock.instant()); + } - MetricsCache(Instant nextUpdateAt) { - this.nextUpdateAt = nextUpdateAt; + void reset() { + nextDiskUsageUpdateAt = clock.instant().minus(Duration.ofDays(1)); + nextRemoveOldFilesAt = clock.instant().minus(Duration.ofDays(1)); + nextHandleOldCoredumpsAt = clock.instant().minus(Duration.ofDays(1)); } } }
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 385e823dc3a..00e0568100d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -33,8 +33,8 @@ import java.util.stream.Collectors; * @author mpolden */ public class AclMaintainer implements Runnable { - private static final PrefixLogger log = PrefixLogger.getNodeAdminLogger(AclMaintainer.class); + private static final PrefixLogger log = PrefixLogger.getNodeAdminLogger(AclMaintainer.class); private static final String IPTABLES_COMMAND = "ip6tables"; private final DockerOperations dockerOperations; @@ -77,7 +77,7 @@ public class AclMaintainer implements Runnable { } } - private void configureAcls() { + private synchronized void configureAcls() { final Map<ContainerName, List<ContainerAclSpec>> aclSpecsGroupedByHostname = nodeRepository .getContainerAclSpecs(nodeAdminHostname).stream() .collect(Collectors.groupingBy(ContainerAclSpec::trustedBy)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 42d5536ea05..a64e7e405d1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.collections.Pair; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.net.HostName; import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; @@ -35,8 +36,10 @@ import java.util.stream.Stream; */ public class NodeAdminImpl implements NodeAdmin { private static final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(NodeAdmin.class); - private final ScheduledExecutorService aclScheduler = Executors.newScheduledThreadPool(1); - private final ScheduledExecutorService metricsScheduler = Executors.newScheduledThreadPool(1); + private final ScheduledExecutorService aclScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("aclscheduler")); + private final ScheduledExecutorService metricsScheduler = + Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 4dee506b9f1..d391f756978 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -16,9 +16,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -39,7 +36,6 @@ public class NodeAdminStateUpdater extends AbstractComponent { private final Object monitor = new Object(); - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); private final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(NodeAdminStateUpdater.class); private Thread loopThread; @@ -135,11 +131,12 @@ public class NodeAdminStateUpdater extends AbstractComponent { * This method attempts to converge node-admin towards one of the {@link State} */ private void convergeState(State wantedState) { - if (wantedState == RESUMED) { - if (!nodeAdmin.setFrozen(false)) { - throw new RuntimeException("NodeAdmin has not yet converged to unfrozen"); - } + boolean wantFrozen = wantedState != RESUMED; + if (!nodeAdmin.setFrozen(wantFrozen)) { + throw new RuntimeException("NodeAdmin has not yet converged to " + (wantFrozen ? "frozen" : "unfrozen")); + } + if (wantedState == RESUMED) { orchestrator.resume(dockerHostHostName); if (wantedState == updateAndGetCurrentState(RESUMED)) return; } @@ -160,10 +157,14 @@ public class NodeAdminStateUpdater extends AbstractComponent { if (currentState == RESUMED) { List<String> nodesToSuspend = new ArrayList<>(nodesInActiveState); nodesToSuspend.add(dockerHostHostName); - orchestrator.suspend(dockerHostHostName, nodesToSuspend); - if (!nodeAdmin.setFrozen(true)) { - throw new RuntimeException("NodeAdmin has not yet converged to frozen"); + try { + orchestrator.suspend(dockerHostHostName, nodesToSuspend); + } catch (RuntimeException e) { + // Because upgrades may take a while, we should unfreeze everything in the mean time + // and try again next tick, since wantedState is still to suspend + nodeAdmin.setFrozen(false); + throw e; } if (wantedState == updateAndGetCurrentState(SUSPENDED_NODE_ADMIN)) return; @@ -182,6 +183,8 @@ public class NodeAdminStateUpdater extends AbstractComponent { private void fetchContainersToRunFromNodeRepository() { synchronized (monitor) { + // Refresh containers to run even if we would like to suspend but have failed to do so yet, + // because it may take a long time to get permission to suspend. if (currentState != RESUMED) { logger.info("Frozen, skipping fetching info from node repository"); return; @@ -237,10 +240,6 @@ public class NodeAdminStateUpdater extends AbstractComponent { if (loopThread.isAlive()) { logger.error("Could not stop NodeAdminStateUpdater tick thread"); } - scheduler.shutdown(); - if (! scheduler.awaitTermination(30, TimeUnit.SECONDS)) { - throw new RuntimeException("Could not stop NodeAdminStateUpdater fetch containers scheduler."); - } } catch (InterruptedException e1) { logger.error("Interrupted; Could not stop NodeAdminStateUpdater thread"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index b4bfaf5c9b0..8e67cac5b2f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; @@ -18,13 +17,8 @@ import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; -import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; import com.yahoo.vespa.hosted.provision.Node; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.time.Clock; import java.time.Duration; @@ -36,7 +30,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.RUNNING; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN; @@ -88,7 +81,6 @@ public class NodeAgentImpl implements NodeAgent { private NodeAttributes lastAttributesSet = null; private ContainerNodeSpec lastNodeSpec = null; private CpuUsageReporter lastCpuMetric; - private Optional<String> vespaVersion = Optional.empty(); public NodeAgentImpl( final String hostName, @@ -118,7 +110,6 @@ public class NodeAgentImpl implements NodeAgent { dockerOperations.getContainer(containerName) .ifPresent(container -> { if (container.state.isRunning()) { - vespaVersion = dockerOperations.getVespaVersion(container.name); lastCpuMetric = new CpuUsageReporter(container.created); } containerState = RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN; @@ -197,27 +188,10 @@ public class NodeAgentImpl implements NodeAgent { } } - private void experimentalWriteFile(final ContainerNodeSpec nodeSpec) { - try { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); - Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec); - if (!config.isPresent()) { - logger.error("Was not able to generate a config for filebeat, ignoring filebeat file creation." + nodeSpec.toString()); - return; - } - Path filebeatPath = environment.pathInNodeAdminFromPathInNode(containerName, "/etc/filebeat/filebeat.yml"); - Files.write(filebeatPath, config.get().getBytes()); - logger.info("Wrote filebeat config."); - } catch (Throwable t) { - logger.error("Failed writing filebeat config; " + nodeSpec, t); - } - } - private void runLocalResumeScriptIfNeeded(final ContainerNodeSpec nodeSpec) { if (containerState != RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN) { return; } - experimentalWriteFile(nodeSpec); addDebugMessage("Starting optional node program resume command"); dockerOperations.resumeNode(containerName); @@ -225,25 +199,14 @@ public class NodeAgentImpl implements NodeAgent { logger.info("Resume command successfully executed, new containerState is " + containerState); } - private void updateNodeRepoAndMarkNodeAsReady(ContainerNodeSpec nodeSpec) { - publishStateToNodeRepoIfChanged( - // Clear current Docker image and vespa version, as nothing is running on this node - new NodeAttributes() - .withRestartGeneration(nodeSpec.wantedRestartGeneration.orElse(null)) - .withRebootGeneration(nodeSpec.wantedRebootGeneration.orElse(0L)) - .withDockerImage(new DockerImage("")) - .withVespaVersion("")); - nodeRepository.markAsReady(nodeSpec.hostname); - } - private void updateNodeRepoWithCurrentAttributes(final ContainerNodeSpec nodeSpec) { final NodeAttributes nodeAttributes = new NodeAttributes() .withRestartGeneration(nodeSpec.wantedRestartGeneration.orElse(null)) // update reboot gen with wanted gen if set, we ignore reboot for Docker nodes but // want the two to be equal in node repo .withRebootGeneration(nodeSpec.wantedRebootGeneration.orElse(0L)) - .withDockerImage(nodeSpec.wantedDockerImage.orElse(new DockerImage(""))) - .withVespaVersion(vespaVersion.orElse("")); + .withDockerImage(nodeSpec.wantedDockerImage.filter(node -> containerState != ABSENT).orElse(new DockerImage(""))) + .withVespaVersion(nodeSpec.wantedVespaVersion.filter(node -> containerState != ABSENT).orElse("")); publishStateToNodeRepoIfChanged(nodeAttributes); } @@ -260,28 +223,29 @@ public class NodeAgentImpl implements NodeAgent { } } - private void startContainerIfNeeded(final ContainerNodeSpec nodeSpec) { - if (!getContainer().isPresent()) { - aclMaintainer.ifPresent(AclMaintainer::run); - dockerOperations.startContainer(containerName, nodeSpec); - metricReceiver.unsetMetricsForContainer(hostname); - lastCpuMetric = new CpuUsageReporter(clock.instant()); - vespaVersion = dockerOperations.getVespaVersion(containerName); - - configureContainerMetrics(nodeSpec); - addDebugMessage("startContainerIfNeeded: containerState " + containerState + " -> " + - RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN); - containerState = RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN; - logger.info("Container successfully started, new containerState is " + containerState); - } + private void startContainer(ContainerNodeSpec nodeSpec) { + aclMaintainer.ifPresent(AclMaintainer::run); + dockerOperations.startContainer(containerName, nodeSpec); + lastCpuMetric = new CpuUsageReporter(clock.instant()); + + writeConfigs(nodeSpec); + + addDebugMessage("startContainerIfNeeded: containerState " + containerState + " -> " + + RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN); + containerState = RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN; + logger.info("Container successfully started, new containerState is " + containerState); } - private void removeContainerIfNeededUpdateContainerState(ContainerNodeSpec nodeSpec) { - removeContainerIfNeeded(nodeSpec).ifPresent(existingContainer -> - shouldRestartServices(nodeSpec).ifPresent(restartReason -> { - logger.info("Will restart services for container " + existingContainer + ": " + restartReason); - restartServices(nodeSpec, existingContainer); - })); + private Optional<Container> removeContainerIfNeededUpdateContainerState(ContainerNodeSpec nodeSpec, Optional<Container> existingContainer) { + return existingContainer + .flatMap(container -> removeContainerIfNeeded(nodeSpec, container)) + .map(container -> { + shouldRestartServices(nodeSpec).ifPresent(restartReason -> { + logger.info("Will restart services for container " + container + ": " + restartReason); + restartServices(nodeSpec, container); + }); + return container; + }); } private Optional<String> shouldRestartServices(ContainerNodeSpec nodeSpec) { @@ -327,16 +291,12 @@ public class NodeAgentImpl implements NodeAgent { return Optional.empty(); } - // Returns true if container is absent on return - private Optional<Container> removeContainerIfNeeded(ContainerNodeSpec nodeSpec) { - Optional<Container> existingContainer = getContainer(); - if (!existingContainer.isPresent()) return Optional.empty(); - - Optional<String> removeReason = shouldRemoveContainer(nodeSpec, existingContainer.get()); + private Optional<Container> removeContainerIfNeeded(ContainerNodeSpec nodeSpec, Container existingContainer) { + Optional<String> removeReason = shouldRemoveContainer(nodeSpec, existingContainer); if (removeReason.isPresent()) { - logger.info("Will remove container " + existingContainer.get() + ": " + removeReason.get()); + logger.info("Will remove container " + existingContainer + ": " + removeReason.get()); - if (existingContainer.get().state.isRunning()) { + if (existingContainer.state.isRunning()) { if (nodeSpec.nodeState == Node.State.active) { orchestratorSuspendNode(); } @@ -347,18 +307,19 @@ public class NodeAgentImpl implements NodeAgent { logger.info("Failed stopping services, ignoring", e); } } - vespaVersion = Optional.empty(); - dockerOperations.removeContainer(existingContainer.get()); + dockerOperations.removeContainer(existingContainer); metricReceiver.unsetMetricsForContainer(hostname); containerState = ABSENT; logger.info("Container successfully removed, new containerState is " + containerState); return Optional.empty(); } - return existingContainer; + return Optional.of(existingContainer); } private void scheduleDownLoadIfNeeded(ContainerNodeSpec nodeSpec) { + if (nodeSpec.currentDockerImage.equals(nodeSpec.wantedDockerImage)) return; + if (dockerOperations.shouldScheduleDownloadOfImage(nodeSpec.wantedDockerImage.get())) { if (nodeSpec.wantedDockerImage.get().equals(imageBeingDownloaded)) { // Downloading already scheduled, but not done. @@ -366,7 +327,7 @@ public class NodeAgentImpl implements NodeAgent { } imageBeingDownloaded = nodeSpec.wantedDockerImage.get(); // Create a signalWorkToBeDone when download is finished. - dockerOperations.scheduleDownloadOfImage(containerName, nodeSpec, this::signalWorkToBeDone); + dockerOperations.scheduleDownloadOfImage(containerName, imageBeingDownloaded, this::signalWorkToBeDone); } else if (imageBeingDownloaded != null) { // Image was downloading, but now it's ready imageBeingDownloaded = null; } @@ -430,13 +391,18 @@ public class NodeAgentImpl implements NodeAgent { .orElseThrow(() -> new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname))); + Optional<Container> container = getContainer(); if (!nodeSpec.equals(lastNodeSpec)) { addDebugMessage("Loading new node spec: " + nodeSpec.toString()); lastNodeSpec = nodeSpec; // Every time the node spec changes, we should clear the metrics for this container as the dimensions // will change and we will be reporting duplicate metrics. + // TODO: Should be retried if writing fails metricReceiver.unsetMetricsForContainer(hostname); + if (container.isPresent()) { + writeConfigs(nodeSpec); + } } switch (nodeSpec.nodeState) { @@ -444,7 +410,7 @@ public class NodeAgentImpl implements NodeAgent { case reserved: case parked: case failed: - removeContainerIfNeededUpdateContainerState(nodeSpec); + removeContainerIfNeededUpdateContainerState(nodeSpec, container); updateNodeRepoWithCurrentAttributes(nodeSpec); break; case active: @@ -453,13 +419,15 @@ public class NodeAgentImpl implements NodeAgent { maintainer.handleCoreDumpsForContainer(containerName, nodeSpec, environment); }); scheduleDownLoadIfNeeded(nodeSpec); - if (imageBeingDownloaded != null) { + if (isDownloadingImage()) { addDebugMessage("Waiting for image to download " + imageBeingDownloaded.asString()); return; } - removeContainerIfNeededUpdateContainerState(nodeSpec); + container = removeContainerIfNeededUpdateContainerState(nodeSpec, container); + if (! container.isPresent()) { + startContainer(nodeSpec); + } - startContainerIfNeeded(nodeSpec); runLocalResumeScriptIfNeeded(nodeSpec); // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes @@ -477,7 +445,7 @@ public class NodeAgentImpl implements NodeAgent { break; case inactive: storageMaintainer.ifPresent(maintainer -> maintainer.removeOldFilesFromNode(containerName)); - removeContainerIfNeededUpdateContainerState(nodeSpec); + removeContainerIfNeededUpdateContainerState(nodeSpec, container); updateNodeRepoWithCurrentAttributes(nodeSpec); break; case provisioned: @@ -485,10 +453,11 @@ public class NodeAgentImpl implements NodeAgent { break; case dirty: storageMaintainer.ifPresent(maintainer -> maintainer.removeOldFilesFromNode(containerName)); - removeContainerIfNeededUpdateContainerState(nodeSpec); + removeContainerIfNeededUpdateContainerState(nodeSpec, container); logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready"); storageMaintainer.ifPresent(maintainer -> maintainer.archiveNodeData(containerName)); - updateNodeRepoAndMarkNodeAsReady(nodeSpec); + updateNodeRepoWithCurrentAttributes(nodeSpec); + nodeRepository.markNodeAvailableForNewAllocation(hostname); break; default: throw new RuntimeException("UNKNOWN STATE " + nodeSpec.nodeState.name()); @@ -507,7 +476,7 @@ public class NodeAgentImpl implements NodeAgent { .add("state", nodeSpec.nodeState.toString()) .add("zone", environment.getZone()) .add("parentHostname", environment.getParentHostHostname()); - vespaVersion.ifPresent(version -> dimensionsBuilder.add("vespaVersion", version)); + nodeSpec.vespaVersion.ifPresent(version -> dimensionsBuilder.add("vespaVersion", version)); nodeSpec.owner.ifPresent(owner -> dimensionsBuilder @@ -522,24 +491,58 @@ public class NodeAgentImpl implements NodeAgent { .add("clustertype", membership.clusterType) .add("clusterid", membership.clusterId)); Dimensions dimensions = dimensionsBuilder.build(); + metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_NODE, dimensions, "alive").sample(1); + // TODO: REMOVE metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.alive").sample(1); - // The remaining metrics require container to exists and be running if (containerState == ABSENT) return; Optional<Docker.ContainerStats> containerStats = dockerOperations.getContainerStats(containerName); if (!containerStats.isPresent()) return; Docker.ContainerStats stats = containerStats.get(); - - long currentCpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue(); - long currentCpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue(); + final String APP = MetricReceiverWrapper.APPLICATION_NODE; + final long bytesInGB = 1 << 30; + final long cpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue(); + final long cpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue(); + final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue(); + final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue(); + final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue(); + final Optional<Long> diskTotalBytes = nodeSpec.minDiskAvailableGb.map(size -> (long) (size * bytesInGB)); + final Optional<Long> diskTotalBytesUsed = storageMaintainer.flatMap(maintainer -> maintainer + .updateIfNeededAndGetDiskMetricsFor(containerName)); // CPU usage by a container is given by dividing used CPU time by the container with CPU time used by the entire // system. Because each container is allocated same amount of CPU shares, no container should use more than 1/n // of the total CPU time, where n is the number of running containers. - double cpuPercentageOfHost = lastCpuMetric.getCpuUsagePercentage(currentCpuContainerTotalTime, currentCpuSystemTotalTime); + double cpuPercentageOfHost = lastCpuMetric.getCpuUsagePercentage(cpuContainerTotalTime, cpuSystemTotalTime); double cpuPercentageOfAllocated = numAllocatedContainersOnHost * cpuPercentageOfHost; + long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache; + double memoryPercentUsed = 100.0 * memoryTotalBytesUsed / memoryTotalBytes; + Optional<Double> diskPercentUsed = diskTotalBytes.flatMap(total -> diskTotalBytesUsed.map(used -> 100.0 * used / total)); + + metricReceiver.declareGauge(APP, dimensions, "cpu.util").sample(cpuPercentageOfAllocated); + metricReceiver.declareGauge(APP, dimensions, "mem.limit").sample(memoryTotalBytes); + metricReceiver.declareGauge(APP, dimensions, "mem.used").sample(memoryTotalBytesUsed); + metricReceiver.declareGauge(APP, dimensions, "mem.util").sample(memoryPercentUsed); + diskTotalBytes.ifPresent(diskLimit -> metricReceiver.declareGauge(APP, dimensions, "disk.limit").sample(diskLimit)); + diskTotalBytesUsed.ifPresent(diskUsed -> metricReceiver.declareGauge(APP, dimensions, "disk.used").sample(diskUsed)); + diskPercentUsed.ifPresent(diskUtil -> metricReceiver.declareGauge(APP, dimensions, "disk.util").sample(diskUtil)); + + stats.getNetworks().forEach((interfaceName, interfaceStats) -> { + Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build(); + Map<String, Number> infStats = (Map<String, Number>) interfaceStats; + + metricReceiver.declareGauge(APP, netDims, "net.in.bytes").sample(infStats.get("rx_bytes").longValue()); + metricReceiver.declareGauge(APP, netDims, "net.in.errors").sample(infStats.get("rx_errors").longValue()); + metricReceiver.declareGauge(APP, netDims, "net.in.dropped").sample(infStats.get("rx_dropped").longValue()); + metricReceiver.declareGauge(APP, netDims, "net.out.bytes").sample(infStats.get("tx_bytes").longValue()); + metricReceiver.declareGauge(APP, netDims, "net.out.errors").sample(infStats.get("tx_errors").longValue()); + metricReceiver.declareGauge(APP, netDims, "net.out.dropped").sample(infStats.get("tx_dropped").longValue()); + }); + + + // TODO: Remove when all alerts and dashboards have been updated to use new metric names metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.cpu.busy.pct").sample(cpuPercentageOfAllocated); addIfNotNull(dimensions, "node.cpu.throttled_time", stats.getCpuStats().get("throttling_data"), "throttled_time"); @@ -561,14 +564,11 @@ public class NodeAgentImpl implements NodeAgent { addIfNotNull(netDims, "node.net.out.dropped", interfaceStats, "tx_dropped"); }); - long bytesInGB = 1 << 30; - nodeSpec.minDiskAvailableGb.ifPresent(diskGB -> metricReceiver - .declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.disk.limit").sample(diskGB * bytesInGB)); - - storageMaintainer.ifPresent(maintainer -> maintainer - .updateIfNeededAndGetDiskMetricsFor(containerName) - .forEach((metricName, metricValue) -> - metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, metricName).sample(metricValue.doubleValue()))); + diskTotalBytes.ifPresent(diskLimit -> + metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.disk.limit").sample(diskLimit)); + diskTotalBytesUsed.ifPresent(diskUsed -> + metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "node.disk.used").sample(diskUsed)); + // TODO END REMOVE metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST_LIFE, dimensions, "uptime").sample(lastCpuMetric.getUptime()); metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST_LIFE, dimensions, "alive").sample(1); @@ -595,6 +595,13 @@ public class NodeAgentImpl implements NodeAgent { } } + private void writeConfigs(ContainerNodeSpec nodeSpec) { + storageMaintainer.ifPresent(maintainer -> { + maintainer.writeMetricsConfig(containerName, nodeSpec); + maintainer.writeFilebeatConfig(containerName, nodeSpec); + }); + } + private Optional<Container> getContainer() { if (containerState == ABSENT) return Optional.empty(); return dockerOperations.getContainer(containerName); @@ -617,40 +624,6 @@ public class NodeAgentImpl implements NodeAgent { return temp; } - private void configureContainerMetrics(ContainerNodeSpec nodeSpec) { - if (!storageMaintainer.isPresent()) return; - final Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, "/etc/yamas-agent/"); - - Path vespaCheckPath = Paths.get(getDefaults().underVespaHome("libexec/yms/yms_check_vespa")); - SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("vespa", 60, vespaCheckPath, "all") - .withTag("namespace", "Vespa") - .withTag("role", "tenants") - .withTag("flavor", nodeSpec.nodeFlavor) - .withTag("state", nodeSpec.nodeState.toString()) - .withTag("zone", environment.getZone()) - .withTag("parentHostname", environment.getParentHostHostname()); - - nodeSpec.owner.ifPresent(owner -> - scheduleMaker - .withTag("tenantName", owner.tenant) - .withTag("app", owner.application + "." + owner.instance)); - - nodeSpec.membership.ifPresent(membership -> - scheduleMaker - .withTag("clustertype", membership.clusterType) - .withTag("clusterid", membership.clusterId)); - - vespaVersion.ifPresent(version -> scheduleMaker.withTag("vespaVersion", version)); - - try { - scheduleMaker.writeTo(yamasAgentFolder); - final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; - dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); - } catch (IOException e) { - throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); - } - } - class CpuUsageReporter { private long totalContainerUsage = 0; private long totalSystemUsage = 0; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java index 8bd55878c64..b2173f31387 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java @@ -23,5 +23,5 @@ public interface NodeRepository { void markAsDirty(String hostName); - void markAsReady(String hostName); + void markNodeAvailableForNewAllocation(String hostName); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java index c3755fc8d5b..554d97ebb3e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java @@ -101,6 +101,7 @@ public class NodeRepositoryImpl implements NodeRepository { Objects.requireNonNull(node.nodeState, "Unknown node state"); Node.State nodeState = Node.State.valueOf(node.nodeState); if (nodeState == Node.State.active) { + Objects.requireNonNull(node.wantedVespaVersion, "Unknown vespa version for active node"); Objects.requireNonNull(node.wantedDockerImage, "Unknown docker image for active node"); Objects.requireNonNull(node.wantedRestartGeneration, "Unknown wantedRestartGeneration for active node"); Objects.requireNonNull(node.currentRestartGeneration, "Unknown currentRestartGeneration for active node"); @@ -122,9 +123,11 @@ public class NodeRepositoryImpl implements NodeRepository { return new ContainerNodeSpec( hostName, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::new), + Optional.ofNullable(node.currentDockerImage).map(DockerImage::new), nodeState, node.nodeType, node.nodeFlavor, + Optional.ofNullable(node.wantedVespaVersion), Optional.ofNullable(node.vespaVersion), Optional.ofNullable(owner), Optional.ofNullable(membership), @@ -153,15 +156,17 @@ public class NodeRepositoryImpl implements NodeRepository { @Override public void markAsDirty(String hostName) { - markNodeToState(hostName, Node.State.dirty); + // This will never happen once the new allocation scheme is rolled out. + markNodeToState(hostName, Node.State.dirty.name()); } @Override - public void markAsReady(final String hostName) { - markNodeToState(hostName, Node.State.ready); + public void markNodeAvailableForNewAllocation(final String hostName) { + // TODO replace with call to delete node when everything has been migrated to dynamic docker allocation + markNodeToState(hostName, "availablefornewallocations"); } - private void markNodeToState(String hostName, Node.State state) { + private void markNodeToState(String hostName, String state) { NodeMessageResponse response = requestExecutor.put( "/nodes/v2/state/" + state + "/" + hostName, port, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java index a63f6f10532..d8d416b9150 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java @@ -31,6 +31,7 @@ public class GetNodesResponse { public final String nodeState; public final String nodeType; public final String nodeFlavor; + public final String wantedVespaVersion; public final String vespaVersion; public final Owner owner; public final Membership membership; @@ -49,6 +50,7 @@ public class GetNodesResponse { @JsonProperty("state") String nodeState, @JsonProperty("type") String nodeType, @JsonProperty("flavor") String nodeFlavor, + @JsonProperty("wantedVespaVersion") String wantedVespaVersion, @JsonProperty("vespaVersion") String vespaVersion, @JsonProperty("owner") Owner owner, @JsonProperty("membership") Membership membership, @@ -65,6 +67,7 @@ public class GetNodesResponse { this.nodeState = nodeState; this.nodeType = nodeType; this.nodeFlavor = nodeFlavor; + this.wantedVespaVersion = wantedVespaVersion; this.vespaVersion = vespaVersion; this.owner = owner; this.membership = membership; @@ -85,6 +88,7 @@ public class GetNodesResponse { + " nodeState = " + nodeState + " nodeType = " + nodeType + " nodeFlavor = " + nodeFlavor + + " wantedVespaVersion = " + wantedVespaVersion + " vespaVersion = " + vespaVersion + " owner = " + owner + " membership = " + membership diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java index 487d1845c62..cce2501a718 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.NodeAdminConfig; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; @@ -50,45 +49,45 @@ public class ComponentsProviderImpl implements ComponentsProvider { // Converge towards desired node admin state every 30 seconds private static final int NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS = 30000; - public ComponentsProviderImpl(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment, - boolean isRunningLocally) { + public ComponentsProviderImpl(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) { String baseHostName = HostName.getLocalhost(); Set<String> configServerHosts = environment.getConfigServerHosts(); if (configServerHosts.isEmpty()) { throw new IllegalStateException("Environment setting for config servers missing or empty."); } + Clock clock = Clock.systemUTC(); ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(configServerHosts); Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor, WEB_SERVICE_PORT, baseHostName); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, metricReceiver); + DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment); - Optional<StorageMaintainer> storageMaintainer = isRunningLocally ? - Optional.empty() : Optional.of(new StorageMaintainer(docker, metricReceiver, environment)); - Optional<AclMaintainer> aclMaintainer = isRunningLocally ? + Optional<StorageMaintainer> storageMaintainer = environment.isRunningLocally() ? + Optional.empty() : Optional.of(new StorageMaintainer(docker, metricReceiver, environment, clock)); + Optional<AclMaintainer> aclMaintainer = environment.isRunningLocally() ? Optional.empty() : Optional.of(new AclMaintainer(dockerOperations, nodeRepository, baseHostName)); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, metricReceiver, environment, Clock.systemUTC(), aclMaintainer); + storageMaintainer, metricReceiver, environment, clock, aclMaintainer); NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver, aclMaintainer); - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, Clock.systemUTC(), orchestrator, baseHostName); + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, clock, orchestrator, baseHostName); nodeAdminStateUpdater.start(NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS); metricReceiverWrapper = metricReceiver; - } - - @Inject - public ComponentsProviderImpl(final NodeAdminConfig config, final Docker docker, final MetricReceiverWrapper metricReceiver) { - this(docker, metricReceiver, new Environment(), config.isRunningLocally()); - if (!config.isRunningLocally()) { + if (! environment.isRunningLocally()) { setCorePattern(docker); initializeNodeAgentSecretAgent(docker); } } + @Inject + public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver) { + this(docker, metricReceiver, new Environment()); + } + @Override public NodeAdminStateUpdater getNodeAdminStateUpdater() { return nodeAdminStateUpdater; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java index 87eb80c1837..aaf45039cb1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -19,16 +18,13 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import javax.ws.rs.core.Response; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * Retries request on config server a few times before giving up. Assumes that all requests should be sent with @@ -59,8 +55,8 @@ public class ConfigServerHttpRequestExecutor { PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // Increase max total connections to 200, which should be enough cm.setMaxTotal(200); - return new ConfigServerHttpRequestExecutor(configServerHosts, HttpClientBuilder.create() - .setConnectionManager(cm).build()); + return new ConfigServerHttpRequestExecutor(configServerHosts, + HttpClientBuilder.create().disableAutomaticRetries().setConnectionManager(cm).build()); } ConfigServerHttpRequestExecutor(Set<String> configServerHosts, CloseableHttpClient client) { @@ -153,16 +149,6 @@ public class ConfigServerHttpRequestExecutor { }, wantedReturnType); } - private static String read(HttpEntity input) { - try { - try (BufferedReader buffer = new BufferedReader(new InputStreamReader(input.getContent()))) { - return buffer.lines().collect(Collectors.joining("\n")); - } - } catch (IOException e) { - return "Failed reading stream: " + e.getMessage(); - } - } - private void setContentTypeToApplicationJson(HttpRequestBase request) { request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index 6264d496643..7f288dccb61 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -16,6 +16,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; @@ -35,6 +36,7 @@ public class Environment { private static final String REGION = "REGION"; private static final String LOGSTASH_NODES = "LOGSTASH_NODES"; private static final String ATHENS_DOMAIN = "ATHENS_DOMAIN"; + private static final String RUNNING_LOCALLY = "RUNNING_LOCALLY"; private final Set<String> configServerHosts; private final String environment; @@ -44,6 +46,7 @@ public class Environment { private final PathResolver pathResolver; private final List<String> logstashNodes; private final String athensDomain; + private final boolean isRunningLocally; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -57,7 +60,8 @@ public class Environment { new InetAddressResolver(), new PathResolver(), getLogstashNodesFromEnvironment(), - getEnvironmentVariable(ATHENS_DOMAIN)); + getEnvironmentVariable(ATHENS_DOMAIN), + Optional.ofNullable(System.getenv(RUNNING_LOCALLY)).map(Boolean::valueOf).orElse(false)); } public Environment(Set<String> configServerHosts, @@ -67,7 +71,8 @@ public class Environment { InetAddressResolver inetAddressResolver, PathResolver pathResolver, List<String> logstashNodes, - String athensDomain) { + String athensDomain, + boolean isRunningLocally) { this.configServerHosts = configServerHosts; this.environment = environment; this.region = region; @@ -76,6 +81,7 @@ public class Environment { this.pathResolver = pathResolver; this.logstashNodes = logstashNodes; this.athensDomain = athensDomain; + this.isRunningLocally = isRunningLocally; } public Set<String> getConfigServerHosts() { return configServerHosts; } @@ -128,6 +134,10 @@ public class Environment { return pathResolver; } + public boolean isRunningLocally() { + return isRunningLocally; + } + /** * Absolute path in node admin to directory with processed and reported core dumps */ @@ -195,6 +205,7 @@ public class Environment { private PathResolver pathResolver; private List<String> logstashNodes = Collections.emptyList(); private String athensDomain; + private boolean isRunningLocally = false; public Builder configServerHosts(String... hosts) { configServerHosts = Arrays.stream(hosts).collect(Collectors.toSet()); @@ -236,9 +247,14 @@ public class Environment { return this; } + public Builder isRunningLocally(boolean isRunningLocally) { + this.isRunningLocally = isRunningLocally; + return this; + } + public Environment build() { return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver, - pathResolver, logstashNodes, athensDomain); + pathResolver, logstashNodes, athensDomain, isRunningLocally); } } } diff --git a/node-admin/src/main/resources/configdefinitions/node-admin.def b/node-admin/src/main/resources/configdefinitions/node-admin.def deleted file mode 100644 index ec0be588868..00000000000 --- a/node-admin/src/main/resources/configdefinitions/node-admin.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.node.admin - -# If true, disables secret-agent, core-dump reporting and container-storage maintenance -isRunningLocally bool default = false
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 2822926945b..5be56a36a5d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -1,25 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.util.Environment; import org.hamcrest.CoreMatchers; import org.junit.Test; import org.mockito.InOrder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.Optional; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyVararg; @@ -32,8 +26,7 @@ import static org.mockito.Mockito.when; public class DockerOperationsImplTest { private final Environment environment = new Environment.Builder().build(); private final Docker docker = mock(Docker.class); - private final DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment, - new MetricReceiverWrapper(MetricReceiver.nullImplementation)); + private final DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment); @Test public void processResultFromNodeProgramWhenSuccess() throws Exception { @@ -107,20 +100,13 @@ public class DockerOperationsImplTest { @Test public void runsCommandInNetworkNamespace() { Container container = makeContainer("container-42", Container.State.RUNNING, 42); - List<String> capturedArgs = new ArrayList<>(); - DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), capturedArgs::addAll); - - dockerOperations.executeCommandInNetworkNamespace(container.name, new String[]{"iptables", "-nvL"}); - - assertEquals(Arrays.asList( - "sudo", - "-n", - "nsenter", - "--net=/host/proc/42/ns/net", - "--", - "iptables", - "-nvL"), capturedArgs); + DockerOperationsImpl dockerOperations = new DockerOperationsImpl(docker, environment); + + when(docker.executeInContainerAsRoot(eq(new ContainerName("node-admin")), eq(60L), + eq("nsenter"), eq("--net=/host/proc/42/ns/net"), eq("--"), eq("iptables"), eq("-nvL"))) + .thenReturn(new ProcessResult(0, "", "")); + + dockerOperations.executeCommandInNetworkNamespace(container.name, "iptables", "-nvL"); } private Container makeContainer(String name, Container.State state, int pid) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java index 80e0b23b55b..fdc5489e95c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java @@ -24,7 +24,6 @@ import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.URL; import java.net.UnknownHostException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; @@ -48,45 +47,47 @@ public class LocalZoneUtils { public static final ContainerName CONFIG_SERVER_CONTAINER_NAME = new ContainerName(CONFIG_SERVER_HOSTNAME); public static final String NODE_ADMIN_HOSTNAME = getParentHostHostname(); public static final ContainerName NODE_ADMIN_CONTAINER_NAME = new ContainerName("node-admin"); - public static final DockerImage VESPA_LOCAL_IMAGE = new DockerImage("vespa-local:latest"); private static final ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create( Collections.singleton(CONFIG_SERVER_HOSTNAME)); private static final String APP_HOSTNAME_PREFIX = "cnode-"; private static final String TENANT_NAME = "localtenant"; private static final String APPLICATION_NAME = "default"; - private static final Path PROJECT_ROOT = Paths.get("").toAbsolutePath(); - public static void startConfigServerIfNeeded(Docker docker, Environment environment) throws UnknownHostException { + public static void startConfigServerIfNeeded(Docker docker, Environment environment, DockerImage dockerImage, Path pathToProjectRoot) throws UnknownHostException { Optional<Container> container = docker.getContainer(CONFIG_SERVER_CONTAINER_NAME); if (container.isPresent()) { if (container.get().state.isRunning()) return; else docker.deleteContainer(CONFIG_SERVER_CONTAINER_NAME); } - docker.createContainerCommand(VESPA_LOCAL_IMAGE, CONFIG_SERVER_CONTAINER_NAME, CONFIG_SERVER_HOSTNAME) + Path pathToConfigServerApp = Paths.get(Defaults.getDefaults().underVespaHome("conf/configserver-app")); + docker.createContainerCommand(dockerImage, CONFIG_SERVER_CONTAINER_NAME, CONFIG_SERVER_HOSTNAME) .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) .withIpAddress(environment.getInetAddressForHost(CONFIG_SERVER_HOSTNAME)) .withEnvironment("HOSTED_VESPA_ENVIRONMENT", environment.getEnvironment()) .withEnvironment("HOSTED_VESPA_REGION", environment.getRegion()) .withEnvironment("CONFIG_SERVER_HOSTNAME", CONFIG_SERVER_HOSTNAME) - .withEntrypoint(Defaults.getDefaults().underVespaHome("bin/start-config-server.sh")) .withUlimit("nofile", 262_144, 262_144) .withUlimit("nproc", 32_768, 409_600) .withUlimit("core", -1, -1) + .withEntrypoint(pathToConfigServerApp.resolve("start-config-server.sh").toString()) .create(); + docker.copyArchiveToContainer(pathToProjectRoot.resolve("node-admin/configserver-app").toString(), + CONFIG_SERVER_CONTAINER_NAME, Defaults.getDefaults().underVespaHome("conf")); + docker.startContainer(CONFIG_SERVER_CONTAINER_NAME); } - public static void startNodeAdminIfNeeded(Docker docker, Environment environment, Path pathToContainerStorage) { + public static void startNodeAdminIfNeeded(Docker docker, Environment environment, DockerImage dockerImage, Path pathToContainerStorage) { Optional<Docker.ContainerStats> containerStats = docker.getContainerStats(NODE_ADMIN_CONTAINER_NAME); if (containerStats.isPresent()) return; else docker.deleteContainer(NODE_ADMIN_CONTAINER_NAME); - Docker.CreateContainerCommand createCmd = docker.createContainerCommand(VESPA_LOCAL_IMAGE, + Docker.CreateContainerCommand createCmd = docker.createContainerCommand(dockerImage, NODE_ADMIN_CONTAINER_NAME, NODE_ADMIN_HOSTNAME) .withNetworkMode("host") .withVolume("/proc", "/host/proc") @@ -94,7 +95,13 @@ public class LocalZoneUtils { .withVolume(pathToContainerStorage.toString(), "/host" + pathToContainerStorage.toString()) .withEnvironment("ENVIRONMENT", environment.getEnvironment()) .withEnvironment("REGION", environment.getRegion()) - .withEnvironment("CONFIG_SERVER_ADDRESS", CONFIG_SERVER_HOSTNAME); + .withEnvironment("CONFIG_SERVER_ADDRESS", CONFIG_SERVER_HOSTNAME) + .withEnvironment("ATHENS_DOMAIN", "fake.env") + .withEnvironment("RUNNING_LOCALLY", "true") + .withUlimit("nofile", 262_144, 262_144) + .withUlimit("nproc", 32_768, 409_600) + .withUlimit("core", -1, -1) + .withEntrypoint("/usr/local/bin/start-services.sh", "--run-local"); if (DockerTestUtils.getSystemOS() == DockerTestUtils.OS.Mac_OS_X) { createCmd.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME); @@ -134,18 +141,7 @@ public class LocalZoneUtils { createCmd.create(); docker.startContainer(NODE_ADMIN_CONTAINER_NAME); - } - - public static void buildVespaLocalDockerImage(Docker docker, DockerImage vespaBaseImage) throws IOException { - Path dockerfilePath = PROJECT_ROOT.resolve("node-admin/include/Dockerfile"); - - Path dockerfileTemplatePath = Paths.get("node-admin/include/Dockerfile.template"); - String dockerfileTemplate = new String(Files.readAllBytes(dockerfileTemplatePath)) - .replaceAll("\\$NODE_ADMIN_FROM_IMAGE", vespaBaseImage.asString()) - .replaceAll("\\$VESPA_HOME", Defaults.getDefaults().vespaHome()); - Files.write(dockerfilePath, dockerfileTemplate.getBytes()); - - docker.buildImage(dockerfilePath.getParent().toFile(), VESPA_LOCAL_IMAGE); + docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "chown", "yahoo", "/host/var/run/docker.sock"); } public static Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName) { @@ -216,9 +212,8 @@ public class LocalZoneUtils { } public static void deployApp(Docker docker, Path pathToApp, String tenantName, String applicationName) { - Path pathToAppOnConfigServer = Paths.get("/tmp"); - docker.copyArchiveToContainer(pathToApp.toAbsolutePath().toString(), - CONFIG_SERVER_CONTAINER_NAME, pathToAppOnConfigServer.toString()); + Path pathToAppOnConfigServer = Paths.get("/tmp").resolve(pathToApp.getFileName()); + docker.copyArchiveToContainer(pathToApp.toString(), CONFIG_SERVER_CONTAINER_NAME, pathToAppOnConfigServer.getParent().toString() + "/"); try { // Add tenant, ignore exception if tenant already exists requestExecutor.put("/application/v2/tenant/" + tenantName, CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class); @@ -230,7 +225,7 @@ public class LocalZoneUtils { System.out.println("prepare " + applicationName); final String deployPath = Defaults.getDefaults().underVespaHome("bin/deploy"); ProcessResult copyProcess = docker.executeInContainer(CONFIG_SERVER_CONTAINER_NAME, deployPath, "-e", - tenantName, "-a", applicationName, "prepare", pathToAppOnConfigServer.resolve(pathToApp.getFileName()).toString()); + tenantName, "-a", applicationName, "prepare", pathToAppOnConfigServer.toString()); if (! copyProcess.isSuccess()) { throw new RuntimeException("Could not prepare " + pathToApp + " on " + CONFIG_SERVER_CONTAINER_NAME.asString() + "\n" + copyProcess.getOutput() + "\n" + copyProcess.getErrors()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java index 5bc72fb8953..249d1b03b0d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java @@ -1,26 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.net.HostName; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.node.admin.util.PathResolver; import com.yahoo.vespa.hosted.provision.Node; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.net.UnknownHostException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.logging.Logger; import java.util.stream.Stream; @@ -38,40 +34,30 @@ import static org.junit.Assert.assertTrue; * @author freva */ public class RunVespaLocal { - private static final Environment.Builder environmentBuilder = new Environment.Builder() + private static final Environment environment = new Environment.Builder() .configServerHosts(LocalZoneUtils.CONFIG_SERVER_HOSTNAME) .environment("dev") .region("vespa-local") .parentHostHostname(HostName.getLocalhost()) - .inetAddressResolver(new InetAddressResolver()); + .inetAddressResolver(new InetAddressResolver()) + .build(); - private final Docker docker; private final Logger logger = Logger.getLogger("RunVespaLocal"); + private final Docker docker; + private final Path pathToVespaRoot; - public RunVespaLocal() { + public RunVespaLocal(Path pathToVespaRoot) { this.docker = DockerTestUtils.getDocker(); + this.pathToVespaRoot = pathToVespaRoot; } - /** - * Pulls the base image and builds the vespa-local image - * @param vespaBaseImage Vespa docker image to use as base for the image that the config-server and nodes will run - */ - public void buildVespaLocalImage(DockerImage vespaBaseImage) throws ExecutionException, InterruptedException, IOException { - if (!docker.imageIsDownloaded(vespaBaseImage)) { - logger.info("Pulling " + vespaBaseImage.asString() + " (This may take a while)"); - docker.pullImageAsync(vespaBaseImage).get(); - } - - logger.info("Building " + LocalZoneUtils.VESPA_LOCAL_IMAGE.asString()); - LocalZoneUtils.buildVespaLocalDockerImage(docker, vespaBaseImage); - } /** * Starts config server, provisions numNodesToProvision and puts them in ready state */ - public void startLocalZoneWithNodes(int numNodesToProvision) throws IOException { + public void startLocalZoneWithNodes(DockerImage dockerImage, int numNodesToProvision) throws IOException { logger.info("Starting config-server"); - LocalZoneUtils.startConfigServerIfNeeded(docker, environmentBuilder.build()); + LocalZoneUtils.startConfigServerIfNeeded(docker, environment, dockerImage, pathToVespaRoot); logger.info("Waiting until config-server is ready to serve"); URL configServerUrl = new URL("http://" + LocalZoneUtils.CONFIG_SERVER_HOSTNAME + @@ -82,7 +68,7 @@ public class RunVespaLocal { Set<String> hostnames = LocalZoneUtils.provisionNodes(LocalZoneUtils.NODE_ADMIN_HOSTNAME, numNodesToProvision); hostnames.stream() .map(LocalZoneUtils::getContainerNodeSpec) - .flatMap(optional -> optional.isPresent() ? Stream.of(optional.get()) : Stream.empty()) // Remove with JDK 9 + .flatMap(optional -> optional.map(Stream::of).orElseGet(Stream::empty)) // Remove with JDK 9 .forEach(nodeSpec -> { if (nodeSpec.nodeState == Node.State.provisioned) LocalZoneUtils.setState(Node.State.dirty, nodeSpec.hostname); if (nodeSpec.nodeState == Node.State.dirty) LocalZoneUtils.setState(Node.State.ready, nodeSpec.hostname); @@ -90,41 +76,37 @@ public class RunVespaLocal { } /** - * Start node-admin in IDE - * @param pathResolver Instance of {@link PathResolver} that specifies the path to where the container data will - * be stored, the path must exist and must be writeable by user, - * normally /home/docker/container-storage - */ - public void startNodeAdminInIDE(PathResolver pathResolver) { - logger.info("Starting node-admin"); - environmentBuilder.pathResolver(pathResolver); - new ComponentsProviderImpl( - docker, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), - environmentBuilder.build(), - true); - } - - /** * Starts node-admin inside a container - * @param pathToNodeAdminApp Path to node-admin application.zip + * @param dockerImage Docker image that node-admin should be running * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must * be writeable by user, normally /home/docker/container-storage */ - public void startNodeAdminAsContainer(Path pathToNodeAdminApp, Path pathToContainerStorage) throws UnknownHostException { + public void startNodeAdminAsContainer(DockerImage dockerImage, Path pathToContainerStorage) throws IOException { logger.info("Starting node-admin"); String parentHostHostname = LocalZoneUtils.NODE_ADMIN_HOSTNAME; - LocalZoneUtils.startNodeAdminIfNeeded(docker, environmentBuilder.build(), pathToContainerStorage); + LocalZoneUtils.startNodeAdminIfNeeded(docker, environment, dockerImage, pathToContainerStorage); logger.info("Provisioning host at " + parentHostHostname); LocalZoneUtils.provisionHost(parentHostHostname); LocalZoneUtils.getContainerNodeSpec(parentHostHostname) .ifPresent(nodeSpec -> { - if (nodeSpec.nodeState == Node.State.provisioned) LocalZoneUtils.setState(Node.State.dirty, nodeSpec.hostname); - if (nodeSpec.nodeState == Node.State.dirty) LocalZoneUtils.setState(Node.State.ready, nodeSpec.hostname); + if (nodeSpec.nodeState == Node.State.provisioned) { + LocalZoneUtils.setState(Node.State.dirty, nodeSpec.hostname); + LocalZoneUtils.setState(Node.State.ready, nodeSpec.hostname); + } }); logger.info("Deploying node-admin app"); + Path pathToNodeAdminApp = pathToVespaRoot.resolve("node-admin/node-admin-zone-app"); + Path pathToNodeAdminAppComponents = pathToNodeAdminApp.resolve("components"); + Files.createDirectories(pathToNodeAdminAppComponents); + Path[] appComponents = {pathToVespaRoot.resolve("node-admin/target/node-admin-jar-with-dependencies.jar"), + pathToVespaRoot.resolve("docker-api/target/docker-api-jar-with-dependencies.jar")}; + + for (Path path : appComponents) { + Files.copy(path, pathToNodeAdminAppComponents.resolve(path.getFileName()), StandardCopyOption.REPLACE_EXISTING); + } + LocalZoneUtils.deployApp(docker, pathToNodeAdminApp, "vespa", "node-admin"); logger.info("Waiting for node-admin to serve"); @@ -149,7 +131,7 @@ public class RunVespaLocal { Set<String> containers = LocalZoneUtils.getContainersForApp(); try { URL nodeUrl = new URL("http://" + containers.iterator().next() + ":" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); - assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120))); + assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofMinutes(3))); logger.info("Endpoint " + nodeUrl + " is now ready"); } catch (MalformedURLException e) { e.printStackTrace(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 223cfd09461..e52052e4cfa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -57,15 +57,16 @@ public class DockerTester implements AutoCloseable { orchestratorMock = new OrchestratorMock(callOrderVerifier); nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); dockerMock = new DockerMock(callOrderVerifier); - StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerMock, environment, callOrderVerifier); + Clock clock = Clock.systemUTC(); + StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerMock, environment, callOrderVerifier, clock); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - final DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, mr); + final DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, - orchestratorMock, dockerOperations, Optional.of(storageMaintainer), mr, environment, Clock.systemUTC(), Optional.empty()); + orchestratorMock, dockerOperations, Optional.of(storageMaintainer), mr, environment, clock, Optional.empty()); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.of(storageMaintainer), 100, mr, Optional.empty()); - updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Clock.systemUTC(), orchestratorMock, "basehostname"); + updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, clock, orchestratorMock, "basehostname"); updater.start(5); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index f497d7cac41..c06febe3f17 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -54,10 +54,10 @@ public class MultiDockerTest { "DeleteContainerStorage with ContainerName { name=host2 }"); callOrderVerifier.assertInOrder( - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion=''}", - "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, vespaVersion=''}", - "markAsReady with HostName: host2.test.yahoo.com", - "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion=''}"); + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion='1.2.3'}", + "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, vespaVersion='1.2.3'}", + "markNodeAvailableForNewAllocation with HostName: host2.test.yahoo.com", + "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion='1.2.3'}"); } } @@ -65,6 +65,7 @@ public class MultiDockerTest { ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec.Builder() .hostname(hostName) .wantedDockerImage(dockerImage) + .wantedVespaVersion("1.2.3") .nodeState(Node.State.active) .nodeType("tenant") .nodeFlavor("docker") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index 9848c80d78a..e0aa2281436 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -79,7 +79,7 @@ public class NodeRepoMock implements NodeRepository { } @Override - public void markAsReady(String hostName) { + public void markNodeAvailableForNewAllocation(String hostName) { Optional<ContainerNodeSpec> cns = getContainerNodeSpec(hostName); synchronized (monitor) { @@ -90,7 +90,7 @@ public class NodeRepoMock implements NodeRepository { .nodeFlavor("docker") .build()); } - callOrderVerifier.add("markAsReady with HostName: " + hostName); + callOrderVerifier.add("markNodeAvailableForNewAllocation with HostName: " + hostName); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index d7fd753fabe..a0024fae3dc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -32,7 +32,7 @@ public class RestartTest { CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier(); // Check that the container is started and NodeRepo has received the PATCH update callOrderVerifier.assertInOrder("createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, vespaVersion=''}"); + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, vespaVersion='1.2.3'}"); wantedRestartGeneration = 2; currentRestartGeneration = 1; @@ -46,8 +46,9 @@ public class RestartTest { private ContainerNodeSpec createContainerNodeSpec(long wantedRestartGeneration, long currentRestartGeneration) { return new ContainerNodeSpec.Builder() .hostname("host1.test.yahoo.com") - .wantedDockerImage(new DockerImage("image:1.2.3")) .nodeState(Node.State.active) + .wantedDockerImage(new DockerImage("image:1.2.3")) + .wantedVespaVersion("1.2.3") .nodeType("tenant") .nodeFlavor("docker") .wantedRestartGeneration(wantedRestartGeneration) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index ddebcdd7a3f..7ec6a55251c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -19,6 +19,7 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.impl.client.HttpClientBuilder; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -28,8 +29,8 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.logging.Logger; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; @@ -40,6 +41,7 @@ import static org.mockito.Mockito.when; * @author dybis */ public class RunInContainerTest { + private final Logger logger = Logger.getLogger("RunInContainerTest"); private final Orchestrator orchestrator = ComponentsProviderWithMocks.orchestratorMock; private final String parentHostname = "localhost.test.yahoo.com"; private JDisc container; @@ -59,19 +61,24 @@ public class RunInContainerTest { doThrow(new RuntimeException()).when(orchestrator).resume(parentHostname); port = findRandomOpenPort(); System.out.println("PORT IS " + port); + logger.info("PORT IS " + port); container = JDisc.fromServicesXml(createServiceXml(port), Networking.enable); } @After - public void after() { - container.close(); + public void stopContainer() { + if (container != null) { + container.close(); + } } private boolean doPutCall(String command) throws IOException { + logger.info("info before '"+command+"' is: " + doGetInfoCall()); HttpClient httpclient = HttpClientBuilder.create().build(); HttpHost target = new HttpHost("localhost", port, "http"); HttpPut getRequest = new HttpPut("/rest/" + command); HttpResponse httpResponse = httpclient.execute(target, getRequest); + logger.info("info after '"+command+"' is: " + doGetInfoCall()); return httpResponse.getStatusLine().getStatusCode() == 200; } @@ -97,6 +104,7 @@ public class RunInContainerTest { if (httpResponse.getStatusLine().getStatusCode() != 200) { continue; } + logger.info("Container started."); System.out.println("Container started."); return; } catch (Exception e) { @@ -106,42 +114,34 @@ public class RunInContainerTest { throw new RuntimeException("Could not get answer from container."); } - @After - public void stopContainer() { - if (container != null) { - container.close(); - } - } - + @Ignore @Test public void testGetContainersToRunAPi() throws IOException, InterruptedException { + doThrow(new OrchestratorException("Cannot suspend because...")).when(orchestrator).suspend(parentHostname); + when(ComponentsProviderWithMocks.nodeRepositoryMock.getContainersToRun()).thenReturn(Collections.emptyList()); waitForJdiscContainerToServe(); - assertFalse(doPutCall("resume")); // Initial is false to force convergence + assertTrue("The initial resume command should fail because it needs to converge first", + verifyWithRetries("resume", false)); doNothing().when(orchestrator).resume(parentHostname); - Thread.sleep(50); - assertTrue(doPutCall("resume")); + assertTrue(verifyWithRetries("resume", true)); - // No nodes are allocated to this host yet, so freezing should be fine, but orchestrator doesnt allow node-admin suspend doThrow(new OrchestratorException("Cannot suspend because...")) .when(orchestrator).suspend(parentHostname, Collections.singletonList(parentHostname)); - assertFalse(doPutCall("suspend/node-admin")); + assertTrue("Should fail because orchestrator does not allow node-admin to suspend", + verifyWithRetries("suspend/node-admin", false)); // Orchestrator changes its mind, allows node-admin to suspend - when(ComponentsProviderWithMocks.nodeRepositoryMock.getContainersToRun()).thenReturn(Collections.emptyList()); doNothing().when(orchestrator).suspend(parentHostname, Collections.singletonList(parentHostname)); - Thread.sleep(50); - assertTrue(doPutCall("suspend/node-admin")); // Tick loop should've run several times by now, expect to be suspended + assertTrue(verifyWithRetries("suspend/node-admin", true)); // Lets try to suspend everything now, should be trivial as we have no active containers to stop services at - assertFalse(doPutCall("suspend")); - Thread.sleep(50); - assertTrue(doPutCall("suspend")); + assertTrue(verifyWithRetries("suspend", false)); + assertTrue(verifyWithRetries("suspend", true)); // Back to resume - assertFalse(doPutCall("resume")); - Thread.sleep(50); - assertTrue(doPutCall("resume")); + assertTrue(verifyWithRetries("resume", false)); + assertTrue(verifyWithRetries("resume", true)); // Lets try the same, but with an active container running on this host when(ComponentsProviderWithMocks.nodeRepositoryMock.getContainersToRun()).thenReturn( @@ -155,17 +155,16 @@ public class RunInContainerTest { doThrow(new OrchestratorException("Cannot suspend because...")).when(orchestrator) .suspend("localhost.test.yahoo.com", Arrays.asList("host1.test.yahoo.com", parentHostname)); - // Orchestrator doesn't allow to suspend either the container or the node-admin - assertFalse(doPutCall("suspend/node-admin")); - Thread.sleep(50); - assertFalse(doPutCall("suspend/node-admin")); + // Initially we are denied to suspend because we have to freeze all the node-agents + assertTrue(verifyWithRetries("suspend/node-admin", false)); + // At this point they should be frozen, but Orchestrator doesn't allow to suspend either the container or the node-admin + assertTrue(verifyWithRetries("suspend/node-admin", false)); doNothing().when(orchestrator) .suspend("localhost.test.yahoo.com", Arrays.asList("host1.test.yahoo.com", parentHostname)); // Orchestrator successfully suspended everything - Thread.sleep(50); - assertTrue(doPutCall("suspend/node-admin")); + assertTrue(verifyWithRetries("suspend/node-admin", true)); // Allow stopping services in active nodes doNothing().when(ComponentsProviderWithMocks.dockerOperationsMock) @@ -173,9 +172,16 @@ public class RunInContainerTest { doNothing().when(ComponentsProviderWithMocks.dockerOperationsMock) .stopServicesOnNode(eq(new ContainerName("host1"))); - assertFalse(doPutCall("suspend")); - Thread.sleep(50); - assertTrue(doPutCall("suspend")); + assertTrue(verifyWithRetries("suspend", false)); + assertTrue(verifyWithRetries("suspend", true)); + } + + private boolean verifyWithRetries(String command, boolean expectedResult) throws IOException, InterruptedException { + for (int i = 0; i < 10; i++) { + if (doPutCall(command) == expectedResult) return true; + Thread.sleep(25); + } + return false; } @@ -192,4 +198,4 @@ public class RunInContainerTest { " </jdisc>\n" + "</services>\n"; } -}
\ No newline at end of file +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index c67b6a0e7e5..30b365c6d0e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.util.Environment; -import java.util.HashMap; -import java.util.Map; +import java.time.Clock; +import java.util.Optional; /** * @author freva @@ -18,14 +18,14 @@ import java.util.Map; public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; - public StorageMaintainerMock(Docker docker, Environment environment, CallOrderVerifier callOrderVerifier) { - super(docker, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment); + public StorageMaintainerMock(Docker docker, Environment environment, CallOrderVerifier callOrderVerifier, Clock clock) { + super(docker, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); this.callOrderVerifier = callOrderVerifier; } @Override - public Map<String, Number> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) { - return new HashMap<>(); + public Optional<Long> updateIfNeededAndGetDiskMetricsFor(ContainerName containerName) { + return Optional.empty(); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 25d267eef28..a4556b5c8bc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -1,8 +1,15 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.util.PathResolver; +import com.yahoo.vespa.hosted.provision.Node; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -10,13 +17,26 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.time.Duration; import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyVararg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * @author dybis */ public class StorageMaintainerTest { + private final ManualClock clock = new ManualClock(); + private final Environment environment = new Environment.Builder() + .pathResolver(new PathResolver()).build(); + private final Docker docker = mock(Docker.class); + private final StorageMaintainer storageMaintainer = new StorageMaintainer(docker, + new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -26,14 +46,78 @@ public class StorageMaintainerTest { int writeSize = 10000; writeNBytesToFile(folder.newFile(), writeSize); - Environment environment = new Environment.Builder().build(); - StorageMaintainer storageMaintainer = new StorageMaintainer(null, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment); long usedBytes = storageMaintainer.getDiscUsedInBytes(folder.getRoot().toPath()); if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4) fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close."); } + @Test + public void testMaintenanceThrottlingAfterSuccessfulMaintenance() { + String hostname = "node-123.us-north-3.test.yahoo.com"; + ContainerName containerName = ContainerName.fromHostname(hostname); + ContainerNodeSpec nodeSpec = new ContainerNodeSpec.Builder() + .hostname(hostname) + .nodeState(Node.State.ready) + .nodeType("tenants") + .nodeFlavor("docker").build(); + + when(docker.executeInContainerAsRoot(any(), anyVararg())).thenReturn(new ProcessResult(0, "", "")); + storageMaintainer.removeOldFilesFromNode(containerName); + verify(docker, times(1)).executeInContainerAsRoot(any(), anyVararg()); + // Will not actually run maintenance job until an hour passes + storageMaintainer.removeOldFilesFromNode(containerName); + verify(docker, times(1)).executeInContainerAsRoot(any(), anyVararg()); + + // Coredump handler has its own throttler + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, environment); + verify(docker, times(2)).executeInContainerAsRoot(any(), anyVararg()); + + + clock.advance(Duration.ofMinutes(61)); + storageMaintainer.removeOldFilesFromNode(containerName); + verify(docker, times(3)).executeInContainerAsRoot(any(), anyVararg()); + + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, environment); + verify(docker, times(4)).executeInContainerAsRoot(any(), anyVararg()); + + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, environment); + verify(docker, times(4)).executeInContainerAsRoot(any(), anyVararg()); + + + // archiveNodeData is unthrottled and it should reset previous times + storageMaintainer.archiveNodeData(containerName); + verify(docker, times(5)).executeInContainerAsRoot(any(), anyVararg()); + storageMaintainer.archiveNodeData(containerName); + verify(docker, times(6)).executeInContainerAsRoot(any(), anyVararg()); + + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, environment); + verify(docker, times(7)).executeInContainerAsRoot(any(), anyVararg()); + } + + @Test + public void testMaintenanceThrottlingAfterFailedMaintenance() { + String hostname = "node-123.us-north-3.test.yahoo.com"; + ContainerName containerName = ContainerName.fromHostname(hostname); + ContainerNodeSpec nodeSpec = new ContainerNodeSpec.Builder() + .hostname(hostname) + .nodeState(Node.State.ready) + .nodeType("tenants") + .nodeFlavor("docker").build(); + + when(docker.executeInContainerAsRoot(any(), anyVararg())) + .thenThrow(new RuntimeException("Something went wrong")) + .thenReturn(new ProcessResult(0, "", "")); + try { + storageMaintainer.removeOldFilesFromNode(containerName); + fail("Maintenance job should've failed!"); + } catch (RuntimeException ignored) { } + verify(docker, times(1)).executeInContainerAsRoot(any(), anyVararg()); + + // Maintenance job failed, we should be able to immediately re-run it + storageMaintainer.removeOldFilesFromNode(containerName); + verify(docker, times(2)).executeInContainerAsRoot(any(), anyVararg()); + } + private static void writeNBytesToFile(File file, int nBytes) throws IOException { Files.write(file.toPath(), new byte[nBytes]); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index f3cc352a3b8..93371c93f73 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -17,6 +17,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyVararg; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -26,6 +27,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class AclMaintainerTest { + private static final String NODE_ADMIN_HOSTNAME = "node-admin.region-1.yahoo.com"; private AclMaintainer aclMaintainer; @@ -95,15 +97,16 @@ public class AclMaintainerTest { doThrow(new RuntimeException("iptables command failed")) .doNothing() .when(dockerOperations) - .executeCommandInNetworkNamespace(any(), any()); + .executeCommandInNetworkNamespace(any(), anyVararg()); aclMaintainer.run(); verify(dockerOperations).executeCommandInNetworkNamespace( eq(container.name), eq("ip6tables"), - eq("-F"), - eq("INPUT") + eq("-P"), + eq("INPUT"), + eq("ACCEPT") ); } @@ -209,4 +212,5 @@ public class AclMaintainerTest { .mapToObj(i -> new ContainerAclSpec("node-" + i, "::" + i, containerName)) .collect(Collectors.toList()); } + } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 2ce7e2a3699..f9b98cb672a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -97,18 +97,19 @@ public class NodeAdminStateUpdaterTest { tickAfter(35); assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); verify(refresher, times(1)).signalWorkToBeDone(); - verify(nodeAdmin, times(1)).setFrozen(eq(false)); + verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Orchestrator denied request to suspend, resume tickAfter(35); assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); - verify(nodeAdmin, times(1)).setFrozen(eq(false)); + verify(nodeAdmin, times(2)).setFrozen(eq(false)); // At this point orchestrator says its OK to suspend, but something goes wrong when we try to stop services doThrow(new RuntimeException("Failed to stop services")).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); tickAfter(0); // Change in wanted state, no need to wait verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state - verify(nodeAdmin, times(1)).setFrozen(eq(false)); // Make sure we dont roll back + // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick + verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Finally we are successful in transitioning to frozen tickAfter(35); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 64c37fedd53..83a00241919 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -93,7 +93,9 @@ public class NodeAgentImplTest { final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.active) + .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration) .currentRestartGeneration(restartGeneration) @@ -102,7 +104,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); nodeAgent.converge(); @@ -131,7 +132,7 @@ public class NodeAgentImplTest { final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) .nodeState(Node.State.active) - .vespaVersion(vespaVersion) + .wantedVespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration) .currentRestartGeneration(restartGeneration) .wantedRebootGeneration(rebootGeneration) @@ -140,7 +141,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); nodeAgent.converge(); @@ -169,7 +169,9 @@ public class NodeAgentImplTest { final long currentRestartGeneration = 1; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(newDockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.active) + .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(wantedRestartGeneration) .currentRestartGeneration(currentRestartGeneration) @@ -188,7 +190,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(dockerOperations); inOrder.verify(dockerOperations, times(1)).shouldScheduleDownloadOfImage(eq(newDockerImage)); - inOrder.verify(dockerOperations, times(1)).scheduleDownloadOfImage(eq(containerName), eq(nodeSpec), any()); + inOrder.verify(dockerOperations, times(1)).scheduleDownloadOfImage(eq(containerName), eq(newDockerImage), any()); } @Test @@ -197,7 +199,9 @@ public class NodeAgentImplTest { final long currentRestartGeneration = 1; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.active) + .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(wantedRestartGeneration) .currentRestartGeneration(currentRestartGeneration) @@ -205,8 +209,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - when(dockerOperations.shouldScheduleDownloadOfImage(any())).thenReturn(false); - try { nodeAgent.converge(); fail("Expected to throw an exception"); @@ -223,7 +225,9 @@ public class NodeAgentImplTest { final long rebootGeneration = 0; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.failed) + .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration) .currentRestartGeneration(restartGeneration) @@ -284,7 +288,9 @@ public class NodeAgentImplTest { final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.inactive) + .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration) .currentRestartGeneration(restartGeneration) @@ -310,6 +316,34 @@ public class NodeAgentImplTest { .withVespaVersion(vespaVersion)); } + @Test + public void reservedNodeDoesNotUpdateNodeRepoWithVersion() throws Exception { + final long restartGeneration = 1; + final long rebootGeneration = 0; + + final ContainerNodeSpec nodeSpec = nodeSpecBuilder + .wantedDockerImage(dockerImage) + .nodeState(Node.State.reserved) + .wantedVespaVersion(vespaVersion) + .wantedRestartGeneration(restartGeneration) + .currentRestartGeneration(restartGeneration) + .wantedRebootGeneration(rebootGeneration) + .build(); + + NodeAgentImpl nodeAgent = makeNodeAgent(null, false); + + when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); + + nodeAgent.converge(); + + verify(nodeRepository).updateNodeAttributes( + hostName, new NodeAttributes() + .withRestartGeneration(restartGeneration) + .withRebootGeneration(rebootGeneration) + .withDockerImage(new DockerImage("")) + .withVespaVersion("")); + } + private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(Node.State nodeState, Optional<Long> wantedRestartGeneration) { wantedRestartGeneration.ifPresent(restartGeneration -> nodeSpecBuilder .wantedRestartGeneration(restartGeneration) @@ -317,6 +351,7 @@ public class NodeAgentImplTest { final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(nodeState) .build(); @@ -330,7 +365,7 @@ public class NodeAgentImplTest { inOrder.verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName)); inOrder.verify(dockerOperations, times(1)).removeContainer(any()); inOrder.verify(storageMaintainer, times(1)).archiveNodeData(eq(containerName)); - inOrder.verify(nodeRepository, times(1)).markAsReady(eq(hostName)); + inOrder.verify(nodeRepository, times(1)).markNodeAvailableForNewAllocation(eq(hostName)); verify(dockerOperations, never()).startContainer(eq(containerName), any()); verify(orchestrator, never()).resume(any(String.class)); @@ -371,6 +406,7 @@ public class NodeAgentImplTest { @Test public void testRestartDeadContainerAfterNodeAdminRestart() throws IOException { final ContainerNodeSpec nodeSpec = nodeSpecBuilder + .currentDockerImage(dockerImage) .wantedDockerImage(dockerImage) .nodeState(Node.State.active) .vespaVersion(vespaVersion) @@ -379,7 +415,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(eq(dockerImage))).thenReturn(false); when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); nodeAgent.tick(); @@ -393,6 +428,7 @@ public class NodeAgentImplTest { final long restartGeneration = 1; final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.active) .vespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration) @@ -402,7 +438,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(eq(dockerImage))).thenReturn(false); final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository); doThrow(new RuntimeException("Failed 1st time")) @@ -475,6 +510,7 @@ public class NodeAgentImplTest { ContainerNodeSpec.Membership membership = new ContainerNodeSpec.Membership("clustType", "clustId", "grp", 3, false); final ContainerNodeSpec nodeSpec = nodeSpecBuilder .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .nodeState(Node.State.active) .vespaVersion(vespaVersion) .owner(owner) @@ -484,7 +520,7 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(eq(dockerImage))).thenReturn(false); + when(storageMaintainer.updateIfNeededAndGetDiskMetricsFor(eq(containerName))).thenReturn(Optional.of(42547019776L)); when(dockerOperations.getContainerStats(eq(containerName))) .thenReturn(Optional.of(stats1)) .thenReturn(Optional.of(stats2)); @@ -516,7 +552,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); - when(dockerOperations.shouldScheduleDownloadOfImage(eq(dockerImage))).thenReturn(false); when(dockerOperations.getContainerStats(eq(containerName))).thenReturn(Optional.empty()); nodeAgent.converge(); // Run the converge loop once to initialize lastNodeSpec @@ -543,8 +578,9 @@ public class NodeAgentImplTest { Optional.empty(); when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats)); - when(dockerOperations.getVespaVersion(eq(containerName))).thenReturn(Optional.of(vespaVersion)); when(dockerOperations.getContainer(eq(containerName))).thenReturn(container); + doNothing().when(storageMaintainer).writeFilebeatConfig(any(), any()); + doNothing().when(storageMaintainer).writeMetricsConfig(any(), any()); return new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, Optional.of(storageMaintainer), metricReceiver, environment, clock, Optional.of(aclMaintainer)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 5c6717075dc..15dce02f336 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -149,17 +149,17 @@ public class NodeRepositoryImplTest { NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost4"); waitForJdiscContainerToServe(); - nodeRepositoryApi.markAsReady("host55.yahoo.com"); + nodeRepositoryApi.markNodeAvailableForNewAllocation("host55.yahoo.com"); try { - nodeRepositoryApi.markAsReady("host1.yahoo.com"); + nodeRepositoryApi.markNodeAvailableForNewAllocation("host1.yahoo.com"); fail("Expected failure because host1 is not registered as provisioned, dirty, failed or parked"); } catch (RuntimeException ignored) { // expected } try { - nodeRepositoryApi.markAsReady("host101.yahoo.com"); + nodeRepositoryApi.markNodeAvailableForNewAllocation("host101.yahoo.com"); fail("Expected failure because host101 does not exist"); } catch (RuntimeException ignored) { // expected diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java index c618b5c4648..660b86fb4ed 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java @@ -48,7 +48,7 @@ public class OrchestratorImplTest { OrchestratorImpl.WEB_SERVICE_PORT, Optional.empty(), UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "service", "fail"))); + )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); orchestrator.suspend(hostName); } @@ -95,7 +95,7 @@ public class OrchestratorImplTest { OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", OrchestratorImpl.WEB_SERVICE_PORT, UpdateHostResponse.class - )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "service", "fail"))); + )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); orchestrator.resume(hostName); } diff --git a/node-admin/src/test/resources/docker.stats.json b/node-admin/src/test/resources/docker.stats.json index d266b01f12d..3b1087b9202 100644 --- a/node-admin/src/test/resources/docker.stats.json +++ b/node-admin/src/test/resources/docker.stats.json @@ -52,7 +52,7 @@ "stats":{ "active_anon":1326051328, "active_file":188919808, - "cache":426680320, + "cache":678965248, "hierarchical_memory_limit":4294967296, "hierarchical_memsw_limit":8589934592, "inactive_anon":0, diff --git a/node-admin/src/test/resources/docker.stats.metrics.active.expected.json b/node-admin/src/test/resources/docker.stats.metrics.active.expected.json index 2ed42cdb7c4..350a054ebac 100644 --- a/node-admin/src/test/resources/docker.stats.metrics.active.expected.json +++ b/node-admin/src/test/resources/docker.stats.metrics.active.expected.json @@ -30,6 +30,106 @@ } }, { + "application": "vespa.node", + "dimensions": { + "flavor": "docker", + "applicationName": "testapp", + "instanceName": "testinstance", + "applicationId": "tester.testapp.testinstance", + "app": "testapp.testinstance", + "clustertype": "clustType", + "role": "tenants", + "tenantName": "tester", + "host": "host1.test.yahoo.com", + "vespaVersion": "1.2.3", + "state": "active", + "clusterid": "clustId", + "parentHostname": "parent.host.name.yahoo.com", + "zone": "dev.us-east-1" + }, + "metrics": { + "alive": 1.0, + "cpu.util": 6.75, + "mem.limit": 4.294967296E9, + "mem.used": 1.073741824E9, + "mem.util": 25.0, + "disk.limit": 2.68435456E11, + "disk.used": 4.2547019776E10, + "disk.util": 15.85 + }, + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } + } + }, + { + "application": "vespa.node", + "dimensions": { + "flavor": "docker", + "applicationName": "testapp", + "instanceName": "testinstance", + "applicationId": "tester.testapp.testinstance", + "app": "testapp.testinstance", + "clustertype": "clustType", + "role": "tenants", + "tenantName": "tester", + "host": "host1.test.yahoo.com", + "vespaVersion": "1.2.3", + "state": "active", + "clusterid": "clustId", + "parentHostname": "parent.host.name.yahoo.com", + "zone": "dev.us-east-1", + "interface": "eth1" + }, + "metrics": { + "net.out.bytes": 5.4246745E7, + "net.out.errors": 0.0, + "net.out.dropped": 0.0, + "net.in.bytes": 3245766.0, + "net.in.errors": 0.0, + "net.in.dropped": 0.0 + }, + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } + } + }, + { + "application": "vespa.node", + "dimensions": { + "flavor": "docker", + "applicationName": "testapp", + "instanceName": "testinstance", + "applicationId": "tester.testapp.testinstance", + "app": "testapp.testinstance", + "clustertype": "clustType", + "role": "tenants", + "tenantName": "tester", + "host": "host1.test.yahoo.com", + "vespaVersion": "1.2.3", + "state": "active", + "clusterid": "clustId", + "parentHostname": "parent.host.name.yahoo.com", + "zone": "dev.us-east-1", + "interface": "eth0" + }, + "metrics": { + "net.out.bytes": 2.0303455E7, + "net.out.errors": 3.0, + "net.out.dropped": 13.0, + "net.in.bytes": 1.949927E7, + "net.in.errors": 55.0, + "net.in.dropped": 4.0 + }, + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } + } + }, + { "application": "docker", "dimensions": { "flavor": "docker", @@ -84,8 +184,9 @@ "node.alive": 1.0, "node.cpu.busy.pct": 6.75, "node.cpu.throttled_time": 4523.0, - "node.memory.usage": 1.326026752E9, + "node.memory.usage": 1.073741824E9, "node.memory.limit": 4.294967296E9, + "node.disk.used": 4.2547019776E10, "node.disk.limit": 2.68435456E11 }, "routing": { diff --git a/node-admin/src/test/resources/docker.stats.metrics.ready.expected.json b/node-admin/src/test/resources/docker.stats.metrics.ready.expected.json index de3e26c8e13..ed2d5e04446 100644 --- a/node-admin/src/test/resources/docker.stats.metrics.ready.expected.json +++ b/node-admin/src/test/resources/docker.stats.metrics.ready.expected.json @@ -1,5 +1,24 @@ [ { + "application": "vespa.node", + "dimensions": { + "flavor": "docker", + "role": "tenants", + "host": "host1.test.yahoo.com", + "state": "ready", + "parentHostname": "parent.host.name.yahoo.com", + "zone": "dev.us-east-1" + }, + "metrics": { + "alive": 1.0 + }, + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } + } + }, + { "application": "docker", "dimensions": { "flavor": "docker", diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java index 7ad2d4c4d66..2179c5b186f 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/FileHelper.java @@ -136,10 +136,8 @@ public class FileHelper { } } - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) { - return listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .max(Comparator.comparing(FileHelper::getLastModifiedTime)); + private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { + return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); } private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java index 5cc3c0280df..22afa59fd2a 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/FileHelperTest.java @@ -192,14 +192,27 @@ public class FileHelperTest { @Test public void testDeleteDirectoriesBasedOnAge() throws IOException { initSubDirectories(); + // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is + // also older than maxAge inside the sub directory, create some files which are newer than maxAge. + // deleteDirectories() should NOT delete folder3 + File subFolder3 = folder.newFolder("test_folder3"); + File subSubFolder3 = folder.newFolder("test_folder3/subSubFolder3"); + + for (int j=0; j<11; j++) { + File.createTempFile("test_", ".json", subSubFolder3); + } + + subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); + subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); //23 files in root // + 13 in test_folder2 // + 13 in subSubFolder2 - // + test_folder2 + subSubFolder2 itself - assertEquals(51, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); + // + 11 in subSubFolder3 + // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself + assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 9c0d504a9f4..86a21806d26 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -5,10 +5,10 @@ import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.Status; @@ -45,10 +45,16 @@ public final class Node { /** The current allocation of this node, if any */ private Optional<Allocation> allocation; - /** Creates a node in the initial state (provisioned) */ + /** Temporary method until we can merge it with the other create method */ + public static Node createDockerNode(String openStackId, Set<String> ipAddresses, Set<String> additionalIpAddresses, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { + return new Node(openStackId, ipAddresses, additionalIpAddresses, hostname, parentHostname, flavor, Status.initial(), State.reserved, + Optional.empty(), History.empty(), type); + } + + /** Creates a node in the initial state (reserved for docker containers, provisioned otherwise) */ public static Node create(String openStackId, Set<String> ipAddresses, Set<String> additionalIpAddresses, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { return new Node(openStackId, ipAddresses, additionalIpAddresses, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type); + Optional.empty(), History.empty(), type); } /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */ @@ -228,7 +234,7 @@ public final class Node { newHistory = history.with(new History.Event(History.Event.Type.rebooted, Agent.system, instant)); return this.with(newStatus).with(newHistory); } - + /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(openStackId, ipAddresses, additionalIpAddresses, hostname, parentHostname, flavor, status, state, allocation, history, type); @@ -256,7 +262,7 @@ public final class Node { throw new IllegalArgumentException(message, e); } } - + @Override public int hashCode() { return id.hashCode(); @@ -300,9 +306,9 @@ public final class Node { /** This node has failed and must be repaired or removed. The node retains any allocation data for diagnosis. */ failed, - /** - * This node should not currently be used. - * This state follows the same rules as failed except that it will never be automatically moved out of + /** + * This node should not currently be used. + * This state follows the same rules as failed except that it will never be automatically moved out of * this state. */ parked; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1f757177022..14d5549d0b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; +import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; @@ -233,15 +234,24 @@ public class NodeRepository extends AbstractComponent { return db.getDefaultFlavorForApplication(applicationId); } + public NodeFlavors getAvailableFlavors() { + return flavors; + } + // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */ - public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Optional<String> parentHostname, + public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Set<String> additionalIpAddresses, Optional<String> parentHostname, Flavor flavor, NodeType type) { if (ipAddresses.isEmpty()) { ipAddresses = nameResolver.getAllByNameOrThrow(hostname); } - return Node.create(openStackId, ImmutableSet.copyOf(ipAddresses), Collections.emptySet(), hostname, parentHostname, flavor, type); + return Node.create(openStackId, ImmutableSet.copyOf(ipAddresses), additionalIpAddresses, hostname, parentHostname, flavor, type); + } + + public Node createNode(String openStackId, String hostname, Set<String> ipAddresses, Optional<String> parentHostname, + Flavor flavor, NodeType type) { + return createNode(openStackId, hostname, ipAddresses, Collections.emptySet(), parentHostname, flavor, type); } public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, @@ -249,6 +259,24 @@ public class NodeRepository extends AbstractComponent { return createNode(openStackId, hostname, Collections.emptySet(), parentHostname, flavor, type); } + /** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */ + public List<Node> addDockerNodes(List<Node> nodes) { + for (Node node : nodes) { + if (!node.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER)) { + throw new IllegalArgumentException("Cannot add " + node.hostname() + ": This is not a docker node"); + } + if (!node.allocation().isPresent()) { + throw new IllegalArgumentException("Cannot add " + node.hostname() + ": Docker containers needs to be allocated"); + } + Optional<Node> existing = getNode(node.hostname()); + if (existing.isPresent()) + throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists"); + } + try (Mutex lock = lockUnallocated()) { + return db.addNodesInState(nodes, Node.State.reserved); + } + } + /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ public List<Node> addNodes(List<Node> nodes) { for (Node node : nodes) { @@ -428,14 +456,23 @@ public class NodeRepository extends AbstractComponent { } /** - * Removes a node. A node must be in the provisioned, failed or parked state before it can be removed. + * Removes a node. A node must be in a legal state before it can be removed. * - * @return true if the node was removed, false if it was not found in one of these states + * @return true if the node was removed, false if it was not found in one of the legal states */ public boolean remove(String hostname) { - Optional<Node> nodeToRemove = getNode(hostname, Node.State.provisioned, Node.State.failed, Node.State.parked); + + Node.State[] legalStates = {Node.State.provisioned, Node.State.failed, Node.State.parked}; + Node.State[] legalDynamicStates = {Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.dirty}; + + Optional<Node> nodeToRemove = getNode(hostname, dynamicAllocationEnabled() ? legalDynamicStates : legalStates); if ( ! nodeToRemove.isPresent()) return false; + // Only docker nodes are allowed to be deleted in state dirty. + if ( nodeToRemove.get().state().equals(Node.State.dirty)) { + if (!(nodeToRemove.get().flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))) return false; + } + try (Mutex lock = lock(nodeToRemove.get())) { return db.removeNode(nodeToRemove.get().state(), hostname); } @@ -547,4 +584,11 @@ public class NodeRepository extends AbstractComponent { return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } + /* + * Temporary feature toggle to enable/disable dynamic docker allocation + * TODO: Remove when enabled in all zones + */ + public boolean dynamicAllocationEnabled() { + return curator.exists(Path.fromString("/provision/v1/dynamicDockerAllocation")); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index a30268f46a0..ff75ee2a40d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -51,7 +51,7 @@ public abstract class Expirer extends Maintainer { expired.add(node); } if ( ! expired.isEmpty()) - log.info(fromState + " expirer found " + expired.size() + " expired nodes"); + log.info(fromState + " expirer found " + expired.size() + " expired nodes: " + expired); expire(expired); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java index 715aaf8680e..3df8187cda8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -18,7 +18,7 @@ import java.util.logging.Logger; */ public abstract class Maintainer extends AbstractComponent implements Runnable { - protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); + protected final Logger log = Logger.getLogger(this.getClass().getName()); private final NodeRepository nodeRepository; private final Duration interval; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index c2991ab2b8b..24896f66385 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -1,18 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity; import java.time.Duration; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * @author oyving @@ -20,6 +20,7 @@ import java.util.logging.Logger; public class MetricsReporter extends Maintainer { private final Metric metric; + private final HashMap<Flavor, Metric.Context> contextMap = new HashMap<>(); public MetricsReporter(NodeRepository nodeRepository, Metric metric, Duration interval, JobControl jobControl) { super(nodeRepository, interval, jobControl); @@ -28,9 +29,33 @@ public class MetricsReporter extends Maintainer { @Override public void maintain() { + // Metrics pr state for (Node.State state : Node.State.values()) metric.set("hostedVespa." + state.name() + "Hosts", nodeRepository().getNodes(NodeType.tenant, state).size(), null); - } + // Capacity flavors for docker + DockerHostCapacity capacity = new DockerHostCapacity(nodeRepository().getNodes(Node.State.values())); + metric.set("hostedVespa.docker.totalCapacityCpu", capacity.getCapacityTotal().getCpu(), null); + metric.set("hostedVespa.docker.totalCapacityMem", capacity.getCapacityTotal().getMemory(), null); + metric.set("hostedVespa.docker.totalCapacityDisk", capacity.getCapacityTotal().getDisk(), null); + metric.set("hostedVespa.docker.freeCapacityCpu", capacity.getFreeCapacityTotal().getCpu(), null); + metric.set("hostedVespa.docker.freeCapacityMem", capacity.getFreeCapacityTotal().getMemory(), null); + metric.set("hostedVespa.docker.freeCapacityDisk", capacity.getFreeCapacityTotal().getDisk(), null); + + List<Flavor> dockerFlavors = nodeRepository().getAvailableFlavors().getFlavors().stream() + .filter(f -> f.getType().equals(Flavor.Type.DOCKER_CONTAINER)) + .collect(Collectors.toList()); + for (Flavor flavor : dockerFlavors) { + if (!contextMap.containsKey(flavor)) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("flavor", flavor.name()); + contextMap.put(flavor, metric.createContext(dimensions)); + } + Metric.Context context = contextMap.get(flavor); + metric.set("hostedVespa.docker.freeCapacityFlavor", capacity.freeCapacityInFlavorEquivalence(flavor), context); + metric.set("hostedVespa.docker.idealHeadroomFlavor", flavor.getIdealHeadroom(), context); + metric.set("hostedVespa.docker.hostsAvailableFlavor", capacity.getNofHostsAvailableFor(flavor), context); + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index 87a55e194a2..fe202aa96ed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -37,6 +37,7 @@ public class NodeRebooter extends Maintainer { // Reboot candidates: Nodes in long-term states, which we know an safely orchestrate a reboot List<Node> rebootCandidates = nodeRepository().getNodes(NodeType.tenant, Node.State.active, Node.State.ready); rebootCandidates.addAll(nodeRepository().getNodes(NodeType.proxy, Node.State.active, Node.State.ready)); + rebootCandidates.addAll(nodeRepository().getNodes(NodeType.host, Node.State.active, Node.State.ready)); for (Node node : rebootCandidates) { if (shouldReboot(node)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index e8672c2003d..d35a9c158b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -13,6 +13,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorClusters; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -39,6 +40,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final ReservationExpirer reservationExpirer; private final InactiveExpirer inactiveExpirer; private final RetiredExpirer retiredExpirer; + private final RetiredEarlyExpirer retiredEarlyExpirer; private final FailedExpirer failedExpirer; private final DirtyExpirer dirtyExpirer; private final NodeRebooter nodeRebooter; @@ -65,16 +67,21 @@ public class NodeRepositoryMaintenance extends AbstractComponent { zooKeeperAccessMaintainer = new ZooKeeperAccessMaintainer(nodeRepository, curator, durationFromEnv("zookeeper_access_maintenance_interval").orElse(defaults.zooKeeperAccessMaintenanceInterval), jobControl); reservationExpirer = new ReservationExpirer(nodeRepository, clock, durationFromEnv("reservation_expiry").orElse(defaults.reservationExpiry), jobControl); retiredExpirer = new RetiredExpirer(nodeRepository, deployer, clock, durationFromEnv("retired_expiry").orElse(defaults.retiredExpiry), jobControl); + retiredEarlyExpirer = new RetiredEarlyExpirer(nodeRepository, zone, durationFromEnv("retired_early_interval").orElse(defaults.retiredEarlyInterval), jobControl, deployer, orchestrator); inactiveExpirer = new InactiveExpirer(nodeRepository, clock, durationFromEnv("inactive_expiry").orElse(defaults.inactiveExpiry), jobControl); failedExpirer = new FailedExpirer(nodeRepository, zone, clock, durationFromEnv("failed_expiry").orElse(defaults.failedExpiry), jobControl); dirtyExpirer = new DirtyExpirer(nodeRepository, clock, durationFromEnv("dirty_expiry").orElse(defaults.dirtyExpiry), jobControl); nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); - nodeRetirer = new NodeRetirer(nodeRepository, zone, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), jobControl, + + FlavorClusters flavorClusters = new FlavorClusters(zone.nodeFlavors().get().getFlavors()); + nodeRetirer = new NodeRetirer(nodeRepository, zone, flavorClusters, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), jobControl, new RetireIPv4OnlyNodes(), new Zone(SystemName.cd, Environment.dev, RegionName.from("cd-us-central-1")), new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-central-1")), - new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-central-2"))); + new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-central-2")), + new Zone(SystemName.main, Environment.perf, RegionName.from("us-east-3")), + new Zone(SystemName.main, Environment.prod, RegionName.from("us-west-1"))); } @Override @@ -86,6 +93,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { reservationExpirer.deconstruct(); inactiveExpirer.deconstruct(); retiredExpirer.deconstruct(); + retiredEarlyExpirer.deconstruct(); failedExpirer.deconstruct(); dirtyExpirer.deconstruct(); nodeRebooter.deconstruct(); @@ -130,6 +138,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration rebootInterval; private final Duration nodeRetirerInterval; private final Duration metricsInterval; + private final Duration retiredEarlyInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -144,6 +153,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { reservationExpiry = Duration.ofMinutes(20); // same as deployment timeout inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredExpiry = Duration.ofDays(4); // enough time to migrate data + retiredEarlyInterval = Duration.ofMinutes(29); failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night dirtyExpiry = Duration.ofHours(2); // enough time to clean the node rebootInterval = Duration.ofDays(30); @@ -160,6 +170,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { reservationExpiry = Duration.ofMinutes(10); // Need to be long enough for deployment to be finished for all config model versions inactiveExpiry = Duration.ofSeconds(2); // support interactive wipe start over retiredExpiry = Duration.ofMinutes(1); + retiredEarlyInterval = Duration.ofMinutes(5); failedExpiry = Duration.ofMinutes(10); dirtyExpiry = Duration.ofMinutes(30); rebootInterval = Duration.ofDays(30); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java index f855eddf1cf..49184dab0af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java @@ -1,15 +1,20 @@ package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorClusters; import java.time.Duration; import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -20,11 +25,14 @@ import java.util.stream.Collectors; * @author freva */ public class NodeRetirer extends Maintainer { + private static final long MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION = 1; private static final Logger log = Logger.getLogger(NodeRetirer.class.getName()); + + private final FlavorClusters flavorClusters; private final RetirementPolicy retirementPolicy; - public NodeRetirer(NodeRepository nodeRepository, Zone zone, Duration interval, JobControl jobControl, - RetirementPolicy retirementPolicy, Zone... applies) { + public NodeRetirer(NodeRepository nodeRepository, Zone zone, FlavorClusters flavorClusters, Duration interval, + JobControl jobControl, RetirementPolicy retirementPolicy, Zone... applies) { super(nodeRepository, interval, jobControl); if (! Arrays.asList(applies).contains(zone)) { String targetZones = Arrays.stream(applies).map(Zone::toString).collect(Collectors.joining(", ")); @@ -33,16 +41,23 @@ public class NodeRetirer extends Maintainer { } this.retirementPolicy = retirementPolicy; + this.flavorClusters = flavorClusters; } @Override protected void maintain() { - retireUnallocated(); + if (retireUnallocated()) { + retireAllocated(); + } } + /** + * Retires unallocated nodes by moving them directly to parked. + * Returns true iff all there are no unallocated nodes that match the retirement policy + */ boolean retireUnallocated() { try (Mutex lock = nodeRepository().lockUnallocated()) { - List<Node> allNodes = nodeRepository().getNodes(); + List<Node> allNodes = nodeRepository().getNodes(NodeType.tenant); Map<Flavor, Long> numSpareNodesByFlavor = getNumberSpareReadyNodesByFlavor(allNodes); long numFlavorsWithUnsuccessfullyRetiredNodes = allNodes.stream() @@ -55,14 +70,97 @@ public class NodeRetirer extends Maintainer { .filter(entry -> { Set<Node> nodesThatShouldBeRetiredForFlavor = entry.getValue(); long numSpareReadyNodesForFlavor = numSpareNodesByFlavor.get(entry.getKey()); - return !limitedPark(nodesThatShouldBeRetiredForFlavor, numSpareReadyNodesForFlavor); + boolean parkedAll = limitedPark(nodesThatShouldBeRetiredForFlavor, numSpareReadyNodesForFlavor); + if (!parkedAll) { + String commaSeparatedHostnames = nodesThatShouldBeRetiredForFlavor.stream().map(Node::hostname) + .collect(Collectors.joining(", ")); + log.info(String.format("Failed to retire %s, wanted to retire %d nodes (%s), but only %d spare " + + "nodes available for flavor cluster.", + entry.getKey(), nodesThatShouldBeRetiredForFlavor.size(), commaSeparatedHostnames, numSpareReadyNodesForFlavor)); + } + return !parkedAll; }).count(); return numFlavorsWithUnsuccessfullyRetiredNodes == 0; } } + void retireAllocated() { + List<Node> allNodes = nodeRepository().getNodes(NodeType.tenant); + List<ApplicationId> activeApplications = getActiveApplicationIds(allNodes); + Map<Flavor, Long> numSpareNodesByFlavor = getNumberSpareReadyNodesByFlavor(allNodes); + + for (ApplicationId applicationId : activeApplications) { + try (Mutex lock = nodeRepository().lock(applicationId)) { + // Get nodes for current application under lock + List<Node> applicationNodes = nodeRepository().getNodes(applicationId); + Set<Node> retireableNodes = getRetireableNodesForApplication(applicationNodes); + long numNodesAllowedToRetire = getNumberNodesAllowToRetireForApplication(applicationNodes, MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION); + + for (Iterator<Node> iterator = retireableNodes.iterator(); iterator.hasNext() && numNodesAllowedToRetire > 0; ) { + Node retireableNode = iterator.next(); + + Set<Flavor> possibleReplacementFlavors = flavorClusters.getFlavorClusterFor(retireableNode.flavor()); + Flavor flavorWithMinSpareNodes = getMinAmongstKeys(numSpareNodesByFlavor, possibleReplacementFlavors); + long spareNodesForMinFlavor = numSpareNodesByFlavor.getOrDefault(flavorWithMinSpareNodes, 0L); + if (spareNodesForMinFlavor > 0) { + log.info("Setting node " + retireableNode + " to wantToRetire and wantToDeprovision. Policy: " + + retirementPolicy.getClass().getSimpleName()); + Node updatedNode = retireableNode + .with(retireableNode.status() + .withWantToRetire(true) + .withWantToDeprovision(true)); + nodeRepository().write(updatedNode); + numSpareNodesByFlavor.put(flavorWithMinSpareNodes, spareNodesForMinFlavor - 1); + numNodesAllowedToRetire--; + } + } + } + } + } + /** + * Returns a list of ApplicationIds sorted by number of active nodes the application has allocated to it + */ + List<ApplicationId> getActiveApplicationIds(List<Node> nodes) { + return nodes.stream() + .filter(node -> node.state() == Node.State.active) + .collect(Collectors.groupingBy( + node -> node.allocation().get().owner(), + Collectors.counting())) + .entrySet().stream() + .sorted((c1, c2) -> c2.getValue().compareTo(c1.getValue())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + /** + * @param applicationNodes All the nodes allocated to an application + * @return Set of nodes that all should eventually be retired + */ + Set<Node> getRetireableNodesForApplication(List<Node> applicationNodes) { + return applicationNodes.stream() + .filter(node -> node.state() == Node.State.active) + .filter(node -> !node.status().wantToRetire()) + .filter(retirementPolicy::shouldRetire) + .collect(Collectors.toSet()); + } + + /** + * @param applicationNodes All the nodes allocated to an application + * @return number of nodes we can safely start retiring + */ + long getNumberNodesAllowToRetireForApplication(List<Node> applicationNodes, long maxSimultaneousRetires) { + long numNodesInWantToRetire = applicationNodes.stream() + .filter(node -> node.status().wantToRetire()) + .filter(node -> node.state() != Node.State.parked) + .count(); + return Math.max(0, maxSimultaneousRetires - numNodesInWantToRetire); + } + + /** + * Parks and sets wantToDeprovision for a subset of size 'limit' of nodes + * * @param nodesToPark Nodes that we want to park * @param limit Maximum number of nodes we want to park * @return True iff we were able to park all the nodes @@ -70,7 +168,10 @@ public class NodeRetirer extends Maintainer { boolean limitedPark(Set<Node> nodesToPark, long limit) { nodesToPark.stream() .limit(limit) - .forEach(node -> nodeRepository().park(node.hostname(), Agent.NodeRetirer, "Policy: " + retirementPolicy.getClass().getSimpleName())); + .forEach(node -> { + nodeRepository().write(node.with(node.status().withWantToDeprovision(true))); + nodeRepository().park(node.hostname(), Agent.NodeRetirer, "Policy: " + retirementPolicy.getClass().getSimpleName()); + }); return limit >= nodesToPark.size(); } @@ -97,7 +198,18 @@ public class NodeRetirer extends Maintainer { * and ready nodes. */ long getNumSpareNodes(long numActiveNodes, long numReadyNodes) { - long numNodesToSpare = (long) Math.ceil(0.1 * numActiveNodes); + long numNodesToSpare = 2; return Math.max(0L, numReadyNodes - numNodesToSpare); } + + /** + * Returns the key with the smallest value amongst keys + */ + <K, V extends Comparable<V>> K getMinAmongstKeys(Map<K, V> map, Set<K> keys) { + return map.entrySet().stream() + .filter(entry -> keys.contains(entry.getKey())) + .min(Comparator.comparing(Map.Entry::getValue)) + .map(Map.Entry::getKey) + .orElseThrow(() -> new RuntimeException("No min key found")); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java new file mode 100644 index 00000000000..78eb0150fb5 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.collections.ListMap; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.orchestrator.OrchestrationException; +import com.yahoo.vespa.orchestrator.Orchestrator; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.stream.Collectors; + +public class RetiredEarlyExpirer extends Maintainer { + private final Deployer deployer; + private final Orchestrator orchestrator; + + public RetiredEarlyExpirer(NodeRepository nodeRepository, + Zone zone, + Duration interval, + JobControl jobControl, + Deployer deployer, + Orchestrator orchestrator) { + super(nodeRepository, interval, jobControl); + this.deployer = deployer; + this.orchestrator = orchestrator; + } + + @Override + protected void maintain() { + List<Node> activeNodes = nodeRepository().getNodes(Node.State.active); + + ListMap<ApplicationId, Node> retiredNodesByApplication = new ListMap<>(); + for (Node node : activeNodes) { + if (node.allocation().isPresent() && node.allocation().get().membership().retired()) { + retiredNodesByApplication.put(node.allocation().get().owner(), node); + } + } + + for (Map.Entry<ApplicationId, List<Node>> entry : retiredNodesByApplication.entrySet()) { + ApplicationId application = entry.getKey(); + List<Node> retiredNodes = entry.getValue(); + + try { + Optional<Deployment> deployment = deployer.deployFromLocalActive(application, Duration.ofMinutes(30)); + if ( ! deployment.isPresent()) continue; // this will be done at another config server + + List<Node> nodesToRemove = new ArrayList<>(); + for (Node node : retiredNodes) { + if (nodeCanBeRemoved(node)) { + nodesToRemove.add(node); + } + } + + if (nodesToRemove.isEmpty()) { + continue; + } + + nodeRepository().setRemovable(application, nodesToRemove); + + deployment.get().activate(); + + String nodeList = nodesToRemove.stream().map(Node::hostname).collect(Collectors.joining(", ")); + log.info("Redeployed " + application + " to deactivate retired nodes: " + nodeList); + } catch (RuntimeException e) { + String nodeList = retiredNodes.stream().map(Node::hostname).collect(Collectors.joining(", ")); + log.log(Level.WARNING, "Exception trying to deactivate retired nodes from " + application + + ": " + nodeList, e); + } + } + } + + boolean nodeCanBeRemoved(Node node) { + try { + orchestrator.acquirePermissionToRemove(new HostName(node.hostname())); + return true; + } catch (OrchestrationException e) { + log.info("Did not get permission to remove retired " + node + ": " + e.getMessage()); + return false; + } + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 49c077f3a05..6b07f8139b4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -71,6 +71,7 @@ public class History { case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); case failed: return this.with(new Event(Event.Type.failed, agent, at)); case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); + case parked: return this.with(new Event(Event.Type.parked, agent, at)); default: return this; } } @@ -111,7 +112,7 @@ public class History { public enum Type { // State move events - readied, reserved, activated, deactivated, deallocated, + readied, reserved, activated, deactivated, deallocated, parked, // The active node was retired retired, // The active node went down according to the service monitor diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 7946b093b10..e8a24b85714 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -22,6 +22,7 @@ public class Status { private final int failCount; private final Optional<HardwareFailureType> hardwareFailure; private final boolean wantToRetire; + private final boolean wantToDeprovision; public enum HardwareFailureType { @@ -40,7 +41,8 @@ public class Status { Optional<String> stateVersion, int failCount, Optional<HardwareFailureType> hardwareFailure, - boolean wantToRetire) { + boolean wantToRetire, + boolean wantToDeprovision) { this.reboot = generation; this.vespaVersion = vespaVersion; this.hostedVersion = hostedVersion; @@ -48,28 +50,29 @@ public class Status { this.failCount = failCount; this.hardwareFailure = hardwareFailure; this.wantToRetire = wantToRetire; + this.wantToDeprovision = wantToDeprovision; } /** Returns a copy of this with the reboot generation changed */ - public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); } + public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns the reboot generation of this node */ public Generation reboot() { return reboot; } /** Returns a copy of this with the vespa version changed */ - public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); } + public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns the Vespa version installed on the node, if known */ public Optional<Version> vespaVersion() { return vespaVersion; } /** Returns a copy of this with the hosted version changed */ - public Status withHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, failCount, hardwareFailure, wantToRetire); } + public Status withHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns the hosted version installed on the node, if known */ public Optional<Version> hostedVersion() { return hostedVersion; } /** Returns a copy of this with the state version changed */ - public Status withStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), failCount, hardwareFailure, wantToRetire); } + public Status withStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** * Returns the state version the node last successfully converged with. @@ -85,29 +88,29 @@ public class Status { .filter(image -> !image.isEmpty()) .map(DockerImage::new) .map(DockerImage::tagAsVersion); - return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); + return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns the current docker image the node is running, if known. */ public Optional<String> dockerImage() { return vespaVersion.map(DockerImage.defaultImage::withTag).map(DockerImage::toString); } - public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount + 1, hardwareFailure, wantToRetire); } + public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount + 1, hardwareFailure, wantToRetire, wantToDeprovision); } - public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount - 1, hardwareFailure, wantToRetire); } + public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount - 1, hardwareFailure, wantToRetire, wantToDeprovision); } - public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, value, hardwareFailure, wantToRetire); } + public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, value, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - public Status withHardwareFailure(Optional<HardwareFailureType> hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); } + public Status withHardwareFailure(Optional<HardwareFailureType> hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** Returns the type of the last hardware failure detected on this node, or empty if none */ public Optional<HardwareFailureType> hardwareFailure() { return hardwareFailure; } /** Returns a copy of this with the want to retire flag changed */ public Status withWantToRetire(boolean wantToRetire) { - return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); + return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); } /** @@ -118,7 +121,19 @@ public class Status { return wantToRetire; } + /** Returns a copy of this with the want to deprovision flag changed */ + public Status withWantToDeprovision(boolean wantToDeprovision) { + return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); + } + + /** + * Returns whether this node should be deprovisioned when possible. + */ + public boolean wantToDeprovision() { + return wantToDeprovision; + } + /** Returns the initial status of a newly provisioned node */ - public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), 0, Optional.empty(), false); } + public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), 0, Optional.empty(), false, false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index e3449bdbc17..582f97361b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; @@ -62,6 +60,7 @@ public class NodeSerializer { private static final String hardwareFailureKey = "hardwareFailure"; private static final String nodeTypeKey = "type"; private static final String wantToRetireKey = "wantToRetire"; + private static final String wantToDeprovisionKey = "wantToDeprovision"; // Configuration fields private static final String flavorKey = "flavor"; @@ -75,7 +74,6 @@ public class NodeSerializer { private static final String currentRestartGenerationKey = "currentRestartGeneration"; private static final String removableKey = "removable"; // Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable. - private static final String dockerImageKey = "dockerImage"; private static final String wantedVespaVersionKey = "wantedVespaVersion"; // History event fields @@ -115,6 +113,7 @@ public class NodeSerializer { object.setLong(failCountKey, node.status().failCount()); node.status().hardwareFailure().ifPresent(failure -> object.setString(hardwareFailureKey, toString(failure))); object.setBool(wantToRetireKey, node.status().wantToRetire()); + object.setBool(wantToDeprovisionKey, node.status().wantToDeprovision()); node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey))); toSlime(node.history(), object.setArray(historyKey)); object.setString(nodeTypeKey, toString(node.type())); @@ -167,13 +166,16 @@ public class NodeSerializer { } private Status statusFromSlime(Inspector object) { + // TODO: Simplify after June 2017 + boolean wantToDeprovision = object.field(wantToDeprovisionKey).valid() && object.field(wantToDeprovisionKey).asBool(); return new Status(generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey), - softwareVersionFromSlime(object.field(vespaVersionKey)), - softwareVersionFromSlime(object.field(hostedVersionKey)), + versionFromSlime(object.field(vespaVersionKey)), + versionFromSlime(object.field(hostedVersionKey)), optionalString(object.field(stateVersionKey)), (int)object.field(failCountKey).asLong(), hardwareFailureFromSlime(object.field(hardwareFailureKey)), - object.field(wantToRetireKey).asBool()); + object.field(wantToRetireKey).asBool(), + wantToDeprovision); } private Flavor flavorFromSlime(Inspector object) { @@ -217,20 +219,12 @@ public class NodeSerializer { return new Generation(object.field(wantedField).asLong(), current.asLong()); } - // TODO: Simplify and inline after April 2017 private ClusterMembership clusterMembershipFromSlime(Inspector object) { - Optional<Version> vespaVersion; - if (object.field(dockerImageKey).valid()) { - vespaVersion = optionalString(object.field(dockerImageKey)) - .map(DockerImage::new) - .map(DockerImage::tagAsVersion); - } else { - vespaVersion = softwareVersionFromSlime(object.field(wantedVespaVersionKey)); - } - return ClusterMembership.from(object.field(serviceIdKey).asString(), vespaVersion.orElse(Vtag.currentVersion)); + return ClusterMembership.from(object.field(serviceIdKey).asString(), + versionFromSlime(object.field(wantedVespaVersionKey)).get()); } - private Optional<Version> softwareVersionFromSlime(Inspector object) { + private Optional<Version> versionFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); return Optional.of(Version.fromString(object.asString())); } @@ -263,6 +257,7 @@ public class NodeSerializer { case "activated" : return History.Event.Type.activated; case "retired" : return History.Event.Type.retired; case "deactivated" : return History.Event.Type.deactivated; + case "parked" : return History.Event.Type.parked; case "failed" : return History.Event.Type.failed; case "deallocated" : return History.Event.Type.deallocated; case "down" : return History.Event.Type.down; @@ -278,6 +273,7 @@ public class NodeSerializer { case activated : return "activated"; case retired : return "retired"; case deactivated : return "deactivated"; + case parked : return "parked"; case failed : return "failed"; case deallocated : return "deallocated"; case down : return "down"; @@ -294,6 +290,7 @@ public class NodeSerializer { case "application" : return Agent.application; case "system" : return Agent.system; case "operator" : return Agent.operator; + case "NodeRetirer" : return Agent.NodeRetirer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } @@ -302,6 +299,7 @@ public class NodeSerializer { case application : return "application"; case system : return "system"; case operator : return "operator"; + case NodeRetirer : return "NodeRetirer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java new file mode 100644 index 00000000000..a027b40ebad --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerAllocator.java @@ -0,0 +1,152 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * Set of methods to allocate new docker nodes + * <p> + * The nodes are not added to the repository here - this is done by caller. + * + * @author smorgrav + */ +public class DockerAllocator { + + /** + * The docker container allocation algorithm + */ + static List<Node> allocateNewDockerNodes(NodeAllocation allocation, + NodeSpec requestedNodes, + List<Node> allNodes, + List<Node> nodesBefore, + NodeFlavors flavors, + Flavor flavor, + int nofSpares, + BiConsumer<List<Node>, String> recorder) { + // Try allocate new nodes with all constraints in place + List<Node> nodesWithHeadroomAndSpares = DockerCapacityConstraints.addHeadroomAndSpareNodes(allNodes, flavors, nofSpares); + recorder.accept(nodesWithHeadroomAndSpares, "Headroom and spares"); + List<Node> accepted = DockerAllocator.allocate(allocation, flavor, nodesWithHeadroomAndSpares); + + List<Node> allNodesIncludingAccepted = new ArrayList<>(allNodes); + allNodesIncludingAccepted.addAll(accepted); + recorder.accept(allNodesIncludingAccepted, "1st dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated - try to allocate the remaining nodes with only hard constraints + if (!allocation.fullfilled()) { + List<Node> nodesWithSpares = DockerCapacityConstraints.addSpareNodes(allNodesIncludingAccepted, nofSpares); + recorder.accept(nodesWithSpares, "Spares only"); + + List<Node> acceptedWithHard = DockerAllocator.allocate(allocation, flavor, nodesWithSpares); + accepted.addAll(acceptedWithHard); + allNodesIncludingAccepted.addAll(acceptedWithHard); + recorder.accept(allNodesIncludingAccepted, "2nd dynamic docker allocation - fullfilled: " + allocation.fullfilled()); + + // If still not fully allocated and this is a replacement - drop all constraints + boolean isReplacement = DockerAllocator.isReplacement(requestedNodes, nodesBefore, allNodes); + if (!allocation.fullfilled() && isReplacement) { + List<Node> finalTry = DockerAllocator.allocate(allocation, flavor, allNodesIncludingAccepted); + accepted.addAll(finalTry); + allNodesIncludingAccepted.addAll(finalTry); + recorder.accept(allNodesIncludingAccepted, "Final dynamic docker alloction - fullfilled: " + allocation.fullfilled()); + } + } + + return accepted; + } + + /** + * Offer the node allocation a prioritized set of new nodes according to capacity constraints + * + * @param allocation The allocation we want to fulfill + * @param flavor Since we create nodes here we need to know the exact flavor + * @param nodes The nodes relevant for the allocation (all nodes from node repo give or take) + * @return Nodes accepted by the node allocation - these nodes does not exist in the noderepo yet. + * @see DockerHostCapacity + */ + public static List<Node> allocate(NodeAllocation allocation, Flavor flavor, List<Node> nodes) { + + DockerHostCapacity dockerCapacity = new DockerHostCapacity(nodes); + + // Get all active docker hosts with enough capacity and ip slots - sorted on free capacity + List<Node> dockerHosts = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> dockerCapacity.hasCapacity(dockerHost, flavor)) + .sorted(dockerCapacity::compare) + .collect(Collectors.toList()); + + // Create one node pr. docker host that we can offer to the allocation + List<Node> offers = new LinkedList<>(); + for (Node parentHost : dockerHosts) { + Set<String> ipAddresses = DockerHostCapacity.findFreeIps(parentHost, nodes); + if (ipAddresses.isEmpty()) continue; + String ipAddress = ipAddresses.stream().findFirst().get(); + String hostname = lookupHostname(ipAddress); + if (hostname == null) continue; + Node node = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), + Collections.emptySet(), hostname, Optional.of(parentHost.hostname()), flavor, NodeType.tenant); + offers.add(node); + } + + return allocation.offer(offers, false); + } + + /** + * From ipAddress - get hostname + * + * @return hostname or null if not able to do the loopup + */ + private static String lookupHostname(String ipAddress) { + try { + return InetAddress.getByName(ipAddress).getHostName(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return null; + } + + /** + * This is an heuristic way to find if new nodes are to replace failing nodes + * or are to expand the cluster. + * + * The current implementation does not account for failed nodes that are not in the application + * anymore. The consequence is that we will ignore the spare capacity constraints too often - in + * particular when the number of failed nodes (not in the application anymore) + * for the cluster equal to the upscaling of the cluster. + * + * The deployment algorithm will still try to allocate the the capacity outside the spare capacity if possible. + * + * TODO propagate this information either through the node object or from the configserver deployer + */ + private static boolean isReplacement(NodeSpec nodeSpec, List<Node> nodesBefore, List<Node> nodesReserved) { + int wantedCount = 0; + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + wantedCount = countSpec.getCount(); + } + + List<Node> failedNodes = new ArrayList<>(); + for (Node node : nodesBefore) { + if (node.state() == Node.State.failed) { + failedNodes.add(node); + } + } + + if (failedNodes.size() == 0) return false; + return (wantedCount <= nodesReserved.size() + failedNodes.size()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java new file mode 100644 index 00000000000..d7cd8bb1e19 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Enforce allocation constraints for docker by manipulating the NodeList we operate on. + * + * The constraints comes in two flavors: headroom and spare. + * + * <b>Headroom</b> is the number of docker nodes (of various flavors) we want to reserve for new applications. + * This is e.g. to make sure we don't smear out small docker flavors on all hosts + * starving allocations for bigger flavors. + * + * <b>Spares</b> is to make sure we have replacement for applications if one or more hosts go down. + * It is more important to safeguard already onboarded applications than accept new applications. + * + * For now - we will use spare also as a means to reserve capacity for future applications that + * have had a separate AI process. + * + * When using spares - we will relay on maintenance jobs to reclaim the spare capacity whenever the + * capacity has been recovered (e.g. when the dead docker host is replaced) + * + * @author smorgrav + */ +public class DockerCapacityConstraints { + + /** This is a static utility class */ + private DockerCapacityConstraints() {} + + /** + * Spare nodes in first iteration is a node that fills up the two + * largest hosts (in terms of free capacity) + */ + public static List<Node> addSpareNodes(List<Node> nodes, int spares) { + DockerHostCapacity capacity = new DockerHostCapacity(nodes); + List<Flavor> spareFlavors = nodes.stream() + .filter(node -> node.type().equals(NodeType.host)) + .filter(dockerHost -> dockerHost.state().equals(Node.State.active)) + .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) + .sorted(capacity::compare) + .limit(spares) + .map(dockerHost -> freeCapacityAsFlavor(dockerHost, nodes)) + .collect(Collectors.toList()); + + return addNodes(nodes, spareFlavors, "spare"); + } + + public static List<Node> addHeadroomAndSpareNodes(List<Node> nodes, NodeFlavors flavors, int nofSpares) { + List<Node> sparesAndHeadroom = addSpareNodes(nodes, nofSpares); + return addNodes(sparesAndHeadroom, flavors.getFlavors(), "headroom"); + } + + private static List<Node> addNodes(List<Node> nodes, List<Flavor> flavors, String id) { + List<Node> headroom = new ArrayList<>(nodes); + for (Flavor flavor : flavors) { + int headroomCount = flavor.getIdealHeadroom(); + if (headroomCount > 0) { + NodeAllocation allocation = createHeadRoomAllocation(flavor, headroomCount, id); + List<Node> acceptedNodes = DockerAllocator.allocate(allocation, flavor, headroom); + headroom.addAll(acceptedNodes); + } + } + return headroom; + } + + private static Flavor freeCapacityAsFlavor(Node host, List<Node> nodes) { + ResourceCapacity hostCapacity = new ResourceCapacity(host); + for (Node container : new NodeList(nodes).childNodes(host).asList()) { + hostCapacity.subtract(container); + } + return hostCapacity.asFlavor(); + } + + private static NodeAllocation createHeadRoomAllocation(Flavor flavor, int count, String id) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, + new ClusterSpec.Id(id), new Version()); + ApplicationId appId = new ApplicationId( + new ApplicationIdConfig( + new ApplicationIdConfig.Builder() + .tenant(id) + .application(id + "-" + flavor.name()) + .instance("temporarynode"))); + + return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor), + new MutableInteger(0), Clock.systemUTC()); + } +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java new file mode 100644 index 00000000000..eeb3149ee53 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -0,0 +1,128 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Capacity calculation for docker hosts. + * <p> + * The calculations is based on an immutable copy of nodes that represents + * all capacities in the system - i.e. all nodes in the node repo give or take. + * + * @author smorgrav + */ +public class DockerHostCapacity { + + /** Tenant name for headroom nodes - only used internally */ + public static final String HEADROOM_TENANT = "-__!@#$$%THISisHEADroom"; + + /** + * An immutable list of nodes + */ + private final NodeList allNodes; + + public DockerHostCapacity(List<Node> allNodes) { + this.allNodes = new NodeList(allNodes); + } + + /** + * Compare hosts on free capacity. + * <p> + * Used in prioritizing hosts for allocation in <b>descending</b> order. + */ + int compare(Node hostA, Node hostB) { + int comp = freeCapacityOf(hostB, true).compare(freeCapacityOf(hostA, true)); + if (comp == 0) { + comp = freeCapacityOf(hostB, false).compare(freeCapacityOf(hostA, false)); + if (comp == 0) { + // If resources are equal - we want to assign to the one with the most IPaddresses free + comp = freeIPs(hostB) - freeIPs(hostA); + } + } + return comp; + } + + /** + * Checks the node capacity and free ip addresses to see + * if we could allocate a flavor on the docker host. + */ + boolean hasCapacity(Node dockerHost, Flavor flavor) { + return freeCapacityOf(dockerHost, true).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; + } + + /** + * Number of free (not allocated) IP addresses assigned to the dockerhost. + */ + int freeIPs(Node dockerHost) { + return findFreeIps(dockerHost, allNodes.asList()).size(); + } + + public ResourceCapacity getFreeCapacityTotal() { + return allNodes.asList().stream() + .filter(n -> n.type().equals(NodeType.host)) + .map(n -> freeCapacityOf(n, false)) + .reduce(new ResourceCapacity(), ResourceCapacity::add); + } + + public ResourceCapacity getCapacityTotal() { + return allNodes.asList().stream() + .filter(n -> n.type().equals(NodeType.host)) + .map(ResourceCapacity::new) + .reduce(new ResourceCapacity(), ResourceCapacity::add); + } + + + public int freeCapacityInFlavorEquivalence(Flavor flavor) { + return allNodes.asList().stream() + .filter(n -> n.type().equals(NodeType.host)) + .map(n -> canFitNumberOf(n, flavor)) + .reduce(0, (a, b) -> a + b); + } + + public long getNofHostsAvailableFor(Flavor flavor) { + return allNodes.asList().stream() + .filter(n -> n.type().equals(NodeType.host)) + .filter(n -> hasCapacity(n, flavor)) + .count(); + } + + private int canFitNumberOf(Node node, Flavor flavor) { + int capacityFactor = freeCapacityOf(node, false).freeCapacityInFlavorEquivalence(flavor); + int ips = freeIPs(node); + return Math.min(capacityFactor, ips); + } + + /** + * Calculate the remaining capacity for the dockerHost. + */ + private ResourceCapacity freeCapacityOf(Node dockerHost, boolean includeHeadroom) { + // Only hosts have free capacity + if (!dockerHost.type().equals(NodeType.host)) return new ResourceCapacity(); + + ResourceCapacity hostCapacity = new ResourceCapacity(dockerHost); + for (Node container : allNodes.childNodes(dockerHost).asList()) { + if (includeHeadroom || !(container.allocation().isPresent() && container.allocation().get().owner().tenant().value().equals(HEADROOM_TENANT))) { + hostCapacity.subtract(container); + } + } + return hostCapacity; + } + + /** + * Compare the additional ip addresses against the set of used addresses on + * child nodes. + */ + static Set<String> findFreeIps(Node dockerHost, List<Node> allNodes) { + Set<String> freeIPAddresses = new HashSet<>(dockerHost.additionalIpAddresses()); + for (Node child : new NodeList(allNodes).childNodes(dockerHost).asList()) { + freeIPAddresses.removeAll(child.ipAddresses()); + } + return freeIPAddresses; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClusters.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClusters.java new file mode 100644 index 00000000000..d21c39581a4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClusters.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Keeps track of flavor clusters: disjoint set of flavors that are connected through 'replaces'. + * Given a node n, which has the flavor x, the flavor cluster of x is the set of flavors that + * n could get next time it is redeployed. + * + * @author freva + */ +public class FlavorClusters { + final Set<Set<Flavor>> flavorClusters; + + public FlavorClusters(List<Flavor> flavors) { + // Make each flavor and its immediate replacements own cluster + Set<Set<Flavor>> prevClusters = flavors.stream() + .map(flavor -> { + Set<Flavor> cluster = new HashSet<>(flavor.replaces()); + cluster.add(flavor); + return cluster; + }).collect(Collectors.toSet()); + + // See if any clusters intersect, if so merge them. Repeat until all the clusters are disjoint. + while (true) { + Set<Set<Flavor>> newClusters = new HashSet<>(); + for (Set<Flavor> oldCluster : prevClusters) { + Optional<Set<Flavor>> overlappingCluster = newClusters.stream() + .filter(cluster -> !Collections.disjoint(cluster, oldCluster)) + .findFirst(); + + if (overlappingCluster.isPresent()) { + overlappingCluster.get().addAll(oldCluster); + } else { + newClusters.add(oldCluster); + } + } + + if (prevClusters.size() == newClusters.size()) break; + prevClusters = newClusters; + } + + flavorClusters = prevClusters; + } + + public Set<Flavor> getFlavorClusterFor(Flavor flavor) { + return flavorClusters.stream() + .filter(cluster -> cluster.contains(flavor)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Could not find cluster for flavor " + flavor)); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index c611438fcbe..474b79af957 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -3,26 +3,20 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.ComparisonChain; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import java.time.Clock; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.Optional; +import java.util.function.BiConsumer; /** * Performs preparation of node activation changes for a single host group in an application. @@ -51,54 +45,89 @@ class GroupPreparer { * This method will remove from this list if it finds it needs additional nodes * @param highestIndex the current highest node index among all active nodes in this cluster. * This method will increase this number when it allocates new nodes to the cluster. + * @param nofSpares The number of spare docker hosts we want when dynamically allocate docker containers + * @param debugRecorder Debug facility to step through the allocation process after the fact * @return the list of nodes this cluster group will have allocated if activated */ // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - List<Node> surplusActiveNodes, MutableInteger highestIndex) { + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, + List<Node> surplusActiveNodes, MutableInteger highestIndex, int nofSpares, BiConsumer<List<Node>, String> debugRecorder) { try (Mutex lock = nodeRepository.lock(application)) { - NodeList nodeList = new NodeList(application, cluster, requestedNodes, highestIndex); + + // A snapshot of nodes before we start the process - used to determind if this is a replacement + List<Node> nodesBefore = nodeRepository.getNodes(application, Node.State.values()); + NodeAllocation allocation = new NodeAllocation(application, cluster, requestedNodes, highestIndex, clock); // Use active nodes - nodeList.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup); - if (nodeList.saturated()) return nodeList.finalNodes(surplusActiveNodes); + allocation.offer(nodeRepository.getNodes(application, Node.State.active), !canChangeGroup); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); // Use active nodes from other groups that will otherwise be retired - List<Node> accepted = nodeList.offer(prioritizeNodes(surplusActiveNodes, requestedNodes), canChangeGroup); + List<Node> accepted = allocation.offer(prioritizeNodes(surplusActiveNodes, requestedNodes), canChangeGroup); surplusActiveNodes.removeAll(accepted); - if (nodeList.saturated()) return nodeList.finalNodes(surplusActiveNodes); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); // Use previously reserved nodes - nodeList.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup); - if (nodeList.saturated()) return nodeList.finalNodes(surplusActiveNodes); + allocation.offer(nodeRepository.getNodes(application, Node.State.reserved), !canChangeGroup); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); // Use inactive nodes - accepted = nodeList.offer(prioritizeNodes(nodeRepository.getNodes(application, Node.State.inactive), requestedNodes), !canChangeGroup); - nodeList.update(nodeRepository.reserve(accepted)); - if (nodeList.saturated()) return nodeList.finalNodes(surplusActiveNodes); + accepted = allocation.offer(prioritizeNodes(nodeRepository.getNodes(application, Node.State.inactive), requestedNodes), !canChangeGroup); + allocation.update(nodeRepository.reserve(accepted)); + if (allocation.saturated()) return allocation.finalNodes(surplusActiveNodes); // Use new, ready nodes. Lock ready pool to ensure that nodes are not grabbed by others. try (Mutex readyLock = nodeRepository.lockUnallocated()) { + + // Check if we have ready nodes that we can allocate List<Node> readyNodes = nodeRepository.getNodes(requestedNodes.type(), Node.State.ready); - accepted = nodeList.offer(stripeOverHosts(prioritizeNodes(readyNodes, requestedNodes)), !canChangeGroup); - nodeList.update(nodeRepository.reserve(accepted)); + accepted = allocation.offer(prioritizeNodes(readyNodes, requestedNodes), !canChangeGroup); + allocation.update(nodeRepository.reserve(accepted)); + + if(nodeRepository.dynamicAllocationEnabled()) { + // Check if we have available capacity on docker hosts that we can allocate + if (!allocation.fullfilled()) { + // The new dynamic allocation method + Optional<Flavor> flavor = getFlavor(requestedNodes); + if (flavor.isPresent() && flavor.get().getType().equals(Flavor.Type.DOCKER_CONTAINER)) { + List<Node> allNodes = nodeRepository.getNodes(Node.State.values()); + NodeFlavors flavors = nodeRepository.getAvailableFlavors(); + accepted = DockerAllocator.allocateNewDockerNodes(allocation, requestedNodes, allNodes, + nodesBefore, flavors, flavor.get(), nofSpares, debugRecorder); + + // Add nodes to the node repository + if (allocation.fullfilled()) { + List<Node> nodesAddedToNodeRepo = nodeRepository.addDockerNodes(accepted); + allocation.update(nodesAddedToNodeRepo); + } + } + } + } } - if (nodeList.fullfilled()) - return nodeList.finalNodes(surplusActiveNodes); + if (allocation.fullfilled()) + return allocation.finalNodes(surplusActiveNodes); else throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + - outOfCapacityDetails(nodeList)); + outOfCapacityDetails(allocation)); } } - - private String outOfCapacityDetails(NodeList nodeList) { - if (nodeList.wouldBeFulfilledWithClashingParentHost()) { + + private Optional<Flavor> getFlavor(NodeSpec nodeSpec) { + if (nodeSpec instanceof NodeSpec.CountNodeSpec) { + NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) nodeSpec; + return Optional.of(countSpec.getFlavor()); + } + return Optional.empty(); + } + + private String outOfCapacityDetails(NodeAllocation allocation) { + if (allocation.wouldBeFulfilledWithClashingParentHost()) { return ": Not enough nodes available on separate physical hosts."; } - if (nodeList.wouldBeFulfilledWithRetiredNodes()) { + if (allocation.wouldBeFulfilledWithRetiredNodes()) { return ": Not enough nodes available due to retirement."; } return "."; @@ -109,17 +138,18 @@ class GroupPreparer { * to use comes first */ private List<Node> prioritizeNodes(List<Node> nodeList, NodeSpec nodeSpec) { - if ( nodeSpec.specifiesNonStockFlavor()) { // sort by exact before inexact flavor match, increasing cost, hostname + if ( nodeSpec.specifiesNonStockFlavor()) { // prefer exact flavor, docker hosts, lower cost, tie break by hostname Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start() .compareTrueFirst(nodeSpec.matchesExactly(n1.flavor()), nodeSpec.matchesExactly(n2.flavor())) + .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent()) .compare(n1.flavor().cost(), n2.flavor().cost()) .compare(n1.hostname(), n2.hostname()) .result() ); } - else { // sort by increasing cost, hostname + else { // prefer docker hosts, lower cost, tie break by hostname Collections.sort(nodeList, (n1, n2) -> ComparisonChain.start() - .compareTrueFirst(nodeSpec.matchesExactly(n1.flavor()), nodeSpec.matchesExactly(n1.flavor())) + .compareTrueFirst(n1.parentHostname().isPresent(), n2.parentHostname().isPresent()) .compare(n1.flavor().cost(), n2.flavor().cost()) .compare(n1.hostname(), n2.hostname()) .result() @@ -127,268 +157,4 @@ class GroupPreparer { } return nodeList; } - - /** Return the input nodes in an order striped over their parent hosts */ - static List<Node> stripeOverHosts(List<Node> input) { - List<Node> output = new ArrayList<>(input.size()); - - // first deal with nodes having a parent host - long nodesHavingParent = input.stream() - .filter(n -> n.parentHostname().isPresent()) - .collect(Collectors.counting()); - if (nodesHavingParent > 0) { - // Make a map where each parent host maps to its list of child nodes - Map<String, List<Node>> byParentHosts = input.stream() - .filter(n -> n.parentHostname().isPresent()) - .collect(Collectors.groupingBy(n -> n.parentHostname().get())); - - // sort keys, those parent hosts with the most (remaining) ready nodes first - List<String> sortedParentHosts = byParentHosts - .keySet().stream() - .sorted((k1, k2) -> byParentHosts.get(k2).size() - byParentHosts.get(k1).size()) - .collect(Collectors.toList()); - while (nodesHavingParent > 0) { - // take one node from each parent host, round-robin. - for (String k : sortedParentHosts) { - List<Node> leftFromHost = byParentHosts.get(k); - if (! leftFromHost.isEmpty()) { - output.add(leftFromHost.remove(0)); - --nodesHavingParent; - } - } - } - } - - // now add non-VMs (nodes without a parent): - input.stream() - .filter(n -> ! n.parentHostname().isPresent()) - .forEach(n -> output.add(n)); - return output; - } - - /** Used to manage a list of nodes during the node reservation process */ - private class NodeList { - - /** The application this list is for */ - private final ApplicationId application; - - /** The cluster this list is for */ - private final ClusterSpec cluster; - - /** The requested nodes of this list */ - private final NodeSpec requestedNodes; - - /** The nodes this has accepted so far */ - private final Set<Node> nodes = new LinkedHashSet<>(); - - /** The number of nodes in the accepted nodes which are of the requested flavor */ - private int acceptedOfRequestedFlavor = 0; - - /** The number of nodes rejected because of clashing parentHostname */ - private int rejectedWithClashingParentHost = 0; - - /** The number of nodes that just now was changed to retired */ - private int wasRetiredJustNow = 0; - - /** The node indexes to verify uniqueness of each members index */ - private Set<Integer> indexes = new HashSet<>(); - - /** The next membership index to assign to a new node */ - private MutableInteger highestIndex; - - public NodeList(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex) { - this.application = application; - this.cluster = cluster; - this.requestedNodes = requestedNodes; - this.highestIndex = highestIndex; - } - - /** - * Offer some nodes to this. The nodes may have an allocation to a different application or cluster, - * an allocation to this cluster, or no current allocation (in which case one is assigned). - * <p> - * Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily - * reject allocated nodes due to index duplicates. - * - * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc. - * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary - * @return the subset of offeredNodes which was accepted, with the correct allocation assigned - */ - public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) { - List<Node> accepted = new ArrayList<>(); - for (Node offered : offeredNodes) { - if (offered.allocation().isPresent()) { - boolean wantToRetireNode = false; - ClusterMembership membership = offered.allocation().get().membership(); - if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application - if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type - if ((! canChangeGroup || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it - if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal - if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) - - // conditions on which we want to retire nodes that were allocated previously - if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) wantToRetireNode = true; - if ( !hasCompatibleFlavor(offered)) wantToRetireNode = true; - if ( offered.flavor().isRetired()) wantToRetireNode = true; - if ( offered.status().wantToRetire()) wantToRetireNode = true; - - if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) - accepted.add(acceptNode(offered, wantToRetireNode)); - } - else if (! saturated() && hasCompatibleFlavor(offered)) { - if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) { - ++rejectedWithClashingParentHost; - continue; - } - if (offered.flavor().isRetired()) { - continue; - } - if (offered.status().wantToRetire()) { - continue; - } - Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant()); - accepted.add(acceptNode(alloc, false)); - } - } - - return accepted; - } - - private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) { - for (Node acceptedNode : accepted) { - if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() && - acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) { - return true; - } - } - return false; - } - - /** - * Returns whether this node should be accepted into the cluster even if it is not currently desired - * (already enough nodes, or wrong flavor). - * Such nodes will be marked retired during finalization of the list of accepted nodes. - * The conditions for this are - * <ul> - * <li>This is a content node. These must always be retired before being removed to allow the cluster to - * migrate away data. - * <li>This is a container node and it is not desired due to having the wrong flavor. In this case this - * will (normally) obtain for all the current nodes in the cluster and so retiring before removing must - * be used to avoid removing all the current nodes at once, before the newly allocated replacements are - * initialized. (In the other case, where a container node is not desired because we have enough nodes we - * do want to remove it immediately to get immediate feedback on how the size reduction works out.) - * </ul> - */ - private boolean acceptToRetire(Node node) { - if (node.state() != Node.State.active) return false; - if (! node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; - - return (cluster.type() == ClusterSpec.Type.content) || - (cluster.type() == ClusterSpec.Type.container && ! hasCompatibleFlavor(node)); - } - - private boolean hasCompatibleFlavor(Node node) { - return requestedNodes.isCompatible(node.flavor()); - } - - /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */ - public void update(List<Node> updatedNodes) { - nodes.removeAll(updatedNodes); - nodes.addAll(updatedNodes); - } - - private Node acceptNode(Node node, boolean wantToRetire) { - if (! wantToRetire) { - if ( ! node.state().equals(Node.State.active)) { - // reactivated node - make sure its not retired - node = node.unretire(); - } - acceptedOfRequestedFlavor++; - } else { - ++wasRetiredJustNow; - // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host - node = node.retire(clock.instant()); - } - if ( ! node.allocation().get().membership().cluster().equals(cluster)) { - // group may be different - node = setCluster(cluster, node); - } - indexes.add(node.allocation().get().membership().index()); - highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); - nodes.add(node); - return node; - } - - private Node setCluster(ClusterSpec cluster, Node node) { - ClusterMembership membership = node.allocation().get().membership().changeCluster(cluster); - return node.with(node.allocation().get().with(membership)); - } - - /** Returns true if no more nodes are needed in this list */ - public boolean saturated() { - return requestedNodes.saturatedBy(acceptedOfRequestedFlavor); - } - - /** Returns true if the content of this list is sufficient to meet the request */ - public boolean fullfilled() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor); - } - - public boolean wouldBeFulfilledWithRetiredNodes() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow); - } - - public boolean wouldBeFulfilledWithClashingParentHost() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost); - } - - /** - * Make the number of <i>non-retired</i> nodes in the list equal to the requested number - * of nodes, and retire the rest of the list. Only retire currently active nodes. - * Prefer to retire nodes of the wrong flavor. - * Make as few changes to the retired set as possible. - * - * @param surplusNodes this will add nodes not any longer needed by this group to this list - * @return the final list of nodes - */ - public List<Node> finalNodes(List<Node> surplusNodes) { - long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count(); - long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired; - - List<Node> changedNodes = new ArrayList<>(); - if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution - for (Node node : byDecreasingIndex(nodes)) { - if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) { - changedNodes.add(node.retire(Agent.application, clock.instant())); - surplusNodes.add(node); // offer this node to other groups - if (--surplus == 0) break; - } - } - } - else if (surplus < 0) { // unretire until surplus is 0 - for (Node node : byIncreasingIndex(nodes)) { - if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) { - changedNodes.add(node.unretire()); - if (++surplus == 0) break; - } - } - } - update(changedNodes); - return new ArrayList<>(nodes); - } - - private List<Node> byDecreasingIndex(Set<Node> nodes) { - return nodes.stream().sorted(nodeIndexComparator().reversed()).collect(Collectors.toList()); - } - - private List<Node> byIncreasingIndex(Set<Node> nodes) { - return nodes.stream().sorted(nodeIndexComparator()).collect(Collectors.toList()); - } - - private Comparator<Node> nodeIndexComparator() { - return Comparator.comparing((Node n) -> n.allocation().get().membership().index()); - } - - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java new file mode 100644 index 00000000000..e149b952fa7 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -0,0 +1,250 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Agent; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Used to manage a list of nodes during the node reservation process + * in order to fulfill the nodespec. + */ +public class NodeAllocation { + + /** The application this list is for */ + private final ApplicationId application; + + /** The cluster this list is for */ + private final ClusterSpec cluster; + + /** The requested nodes of this list */ + private final NodeSpec requestedNodes; + + /** The nodes this has accepted so far */ + private final Set<Node> nodes = new LinkedHashSet<>(); + + /** The number of nodes in the accepted nodes which are of the requested flavor */ + private int acceptedOfRequestedFlavor = 0; + + /** The number of nodes rejected because of clashing parentHostname */ + private int rejectedWithClashingParentHost = 0; + + /** The number of nodes that just now was changed to retired */ + private int wasRetiredJustNow = 0; + + /** The node indexes to verify uniqueness of each members index */ + private final Set<Integer> indexes = new HashSet<>(); + + /** The next membership index to assign to a new node */ + private final MutableInteger highestIndex; + + /** Used to record event timestamps **/ + private final Clock clock; + + public NodeAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, MutableInteger highestIndex, Clock clock) { + this.application = application; + this.cluster = cluster; + this.requestedNodes = requestedNodes; + this.highestIndex = highestIndex; + this.clock = clock; + } + + /** + * Offer some nodes to this. The nodes may have an allocation to a different application or cluster, + * an allocation to this cluster, or no current allocation (in which case one is assigned). + * <p> + * Note that if unallocated nodes are offered before allocated nodes, this will unnecessarily + * reject allocated nodes due to index duplicates. + * + * @param offeredNodes the nodes which are potentially on offer. These may belong to a different application etc. + * @param canChangeGroup whether it is ok to change the group the offered node is to belong to if necessary + * @return the subset of offeredNodes which was accepted, with the correct allocation assigned + */ + public List<Node> offer(List<Node> offeredNodes, boolean canChangeGroup) { + List<Node> accepted = new ArrayList<>(); + for (Node offered : offeredNodes) { + if (offered.allocation().isPresent()) { + boolean wantToRetireNode = false; + ClusterMembership membership = offered.allocation().get().membership(); + if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application + if ( ! membership.cluster().equalsIgnoringGroupAndVespaVersion(cluster)) continue; // wrong cluster id/type + if ((! canChangeGroup || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it + if ( offered.allocation().get().isRemovable()) continue; // don't accept; causes removal + if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) + + // conditions on which we want to retire nodes that were allocated previously + if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) wantToRetireNode = true; + if ( !hasCompatibleFlavor(offered)) wantToRetireNode = true; + if ( offered.flavor().isRetired()) wantToRetireNode = true; + if ( offered.status().wantToRetire()) wantToRetireNode = true; + + if ((!saturated() && hasCompatibleFlavor(offered)) || acceptToRetire(offered) ) + accepted.add(acceptNode(offered, wantToRetireNode)); + } + else if (! saturated() && hasCompatibleFlavor(offered)) { + if ( offeredNodeHasParentHostnameAlreadyAccepted(this.nodes, offered)) { + ++rejectedWithClashingParentHost; + continue; + } + if (offered.flavor().isRetired()) { + continue; + } + if (offered.status().wantToRetire()) { + continue; + } + Node alloc = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), clock.instant()); + accepted.add(acceptNode(alloc, false)); + } + } + + return accepted; + } + + private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<Node> accepted, Node offered) { + for (Node acceptedNode : accepted) { + if (acceptedNode.parentHostname().isPresent() && offered.parentHostname().isPresent() && + acceptedNode.parentHostname().get().equals(offered.parentHostname().get())) { + return true; + } + } + return false; + } + + /** + * Returns whether this node should be accepted into the cluster even if it is not currently desired + * (already enough nodes, or wrong flavor). + * Such nodes will be marked retired during finalization of the list of accepted nodes. + * The conditions for this are + * <ul> + * <li>This is a content node. These must always be retired before being removed to allow the cluster to + * migrate away data. + * <li>This is a container node and it is not desired due to having the wrong flavor. In this case this + * will (normally) obtain for all the current nodes in the cluster and so retiring before removing must + * be used to avoid removing all the current nodes at once, before the newly allocated replacements are + * initialized. (In the other case, where a container node is not desired because we have enough nodes we + * do want to remove it immediately to get immediate feedback on how the size reduction works out.) + * </ul> + */ + private boolean acceptToRetire(Node node) { + if (node.state() != Node.State.active) return false; + if (! node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; + + return (cluster.type() == ClusterSpec.Type.content) || + (cluster.type() == ClusterSpec.Type.container && ! hasCompatibleFlavor(node)); + } + + private boolean hasCompatibleFlavor(Node node) { + return requestedNodes.isCompatible(node.flavor()); + } + + /** Updates the state of some existing nodes in this list by replacing them by id with the given instances. */ + public void update(List<Node> updatedNodes) { + nodes.removeAll(updatedNodes); + nodes.addAll(updatedNodes); + } + + private Node acceptNode(Node node, boolean wantToRetire) { + if (! wantToRetire) { + if ( ! node.state().equals(Node.State.active)) { + // reactivated node - make sure its not retired + node = node.unretire(); + } + acceptedOfRequestedFlavor++; + } else { + ++wasRetiredJustNow; + // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host + node = node.retire(clock.instant()); + } + if ( ! node.allocation().get().membership().cluster().equals(cluster)) { + // group may be different + node = setCluster(cluster, node); + } + indexes.add(node.allocation().get().membership().index()); + highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); + nodes.add(node); + return node; + } + + private Node setCluster(ClusterSpec cluster, Node node) { + ClusterMembership membership = node.allocation().get().membership().changeCluster(cluster); + return node.with(node.allocation().get().with(membership)); + } + + /** Returns true if no more nodes are needed in this list */ + public boolean saturated() { + return requestedNodes.saturatedBy(acceptedOfRequestedFlavor); + } + + /** Returns true if the content of this list is sufficient to meet the request */ + public boolean fullfilled() { + return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor); + } + + public boolean wouldBeFulfilledWithRetiredNodes() { + return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow); + } + + public boolean wouldBeFulfilledWithClashingParentHost() { + return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost); + } + + /** + * Make the number of <i>non-retired</i> nodes in the list equal to the requested number + * of nodes, and retire the rest of the list. Only retire currently active nodes. + * Prefer to retire nodes of the wrong flavor. + * Make as few changes to the retired set as possible. + * + * @param surplusNodes this will add nodes not any longer needed by this group to this list + * @return the final list of nodes + */ + public List<Node> finalNodes(List<Node> surplusNodes) { + long currentRetired = nodes.stream().filter(node -> node.allocation().get().membership().retired()).count(); + long surplus = requestedNodes.surplusGiven(nodes.size()) - currentRetired; + + List<Node> changedNodes = new ArrayList<>(); + if (surplus > 0) { // retire until surplus is 0, prefer to retire higher indexes to minimize redistribution + for (Node node : byDecreasingIndex(nodes)) { + if ( ! node.allocation().get().membership().retired() && node.state().equals(Node.State.active)) { + changedNodes.add(node.retire(Agent.application, clock.instant())); + surplusNodes.add(node); // offer this node to other groups + if (--surplus == 0) break; + } + } + } + else if (surplus < 0) { // unretire until surplus is 0 + for (Node node : byIncreasingIndex(nodes)) { + if ( node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) { + changedNodes.add(node.unretire()); + if (++surplus == 0) break; + } + } + } + update(changedNodes); + return new ArrayList<>(nodes); + } + + private List<Node> byDecreasingIndex(Set<Node> nodes) { + return nodes.stream().sorted(nodeIndexComparator().reversed()).collect(Collectors.toList()); + } + + private List<Node> byIncreasingIndex(Set<Node> nodes) { + return nodes.stream().sorted(nodeIndexComparator()).collect(Collectors.toList()); + } + + private Comparator<Node> nodeIndexComparator() { + return Comparator.comparing((Node n) -> n.allocation().get().membership().index()); + } + +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 1feb12ba9ab..bac99b2bab4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -17,7 +18,6 @@ import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -39,24 +40,30 @@ import java.util.logging.Logger; public class NodeRepositoryProvisioner implements Provisioner { private static Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName()); + private static final int SPARE_CAPACITY_PROD = 2; + private static final int SPARE_CAPACITY_NONPROD = 0; private final NodeRepository nodeRepository; private final CapacityPolicies capacityPolicies; private final Zone zone; private final Preparer preparer; private final Activator activator; + private final BiConsumer<List<Node>, String> debugRecorder; @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone) { - this(nodeRepository, flavors, zone, Clock.systemUTC()); + this(nodeRepository, flavors, zone, Clock.systemUTC(), (x,y) -> {}); } - public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock) { + public NodeRepositoryProvisioner(NodeRepository nodeRepository, NodeFlavors flavors, Zone zone, Clock clock, BiConsumer<List<Node>, String> debugRecorder) { this.nodeRepository = nodeRepository; this.capacityPolicies = new CapacityPolicies(zone, flavors); this.zone = zone; - this.preparer = new Preparer(nodeRepository, clock); + this.preparer = new Preparer(nodeRepository, clock, zone.environment().equals(Environment.prod) + ? SPARE_CAPACITY_PROD + : SPARE_CAPACITY_NONPROD); this.activator = new Activator(nodeRepository, clock); + this.debugRecorder = debugRecorder; } /** @@ -92,7 +99,7 @@ public class NodeRepositoryProvisioner implements Provisioner { effectiveGroups = 1; // type request with multiple groups is not supported } - return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups)); + return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups, debugRecorder)); } @Override @@ -121,5 +128,4 @@ public class NodeRepositoryProvisioner implements Provisioner { } return hosts; } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index cb6786270e7..ab4012b7d3e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -58,6 +58,12 @@ public interface NodeSpec { this.flavor = flavor; } + public Flavor getFlavor() { + return flavor; + } + + public int getCount() { return count; } + @Override public NodeType type() { return NodeType.tenant; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 57db9707049..dfd7be7804a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.List; import java.util.ListIterator; import java.util.Optional; +import java.util.function.BiConsumer; /** * Performs preparation of node activation changes for an application. @@ -25,10 +26,12 @@ class Preparer { private final NodeRepository nodeRepository; private final Clock clock; private final GroupPreparer groupPreparer; + private final int nofSpares; - public Preparer(NodeRepository nodeRepository, Clock clock) { + public Preparer(NodeRepository nodeRepository, Clock clock, int nofSpares) { this.nodeRepository = nodeRepository; this.clock = clock; + this.nofSpares = nofSpares; groupPreparer = new GroupPreparer(nodeRepository, clock); } @@ -40,7 +43,7 @@ class Preparer { // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups, BiConsumer<List<Node>, String> debugRecorder) { List<Node> surplusNodes = findNodesInRemovableGroups(application, cluster, wantedGroups); MutableInteger highestIndex = new MutableInteger(findHighestIndex(application, cluster)); @@ -48,7 +51,7 @@ class Preparer { for (int groupIndex = 0; groupIndex < wantedGroups; groupIndex++) { ClusterSpec clusterGroup = cluster.changeGroup(Optional.of(ClusterSpec.Group.from(groupIndex))); List<Node> accepted = groupPreparer.prepare(application, clusterGroup, - requestedNodes.fraction(wantedGroups), surplusNodes, highestIndex); + requestedNodes.fraction(wantedGroups), surplusNodes, highestIndex, nofSpares, debugRecorder); replace(acceptedNodes, accepted); } moveToActiveGroup(surplusNodes, wantedGroups, cluster.group()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java new file mode 100644 index 00000000000..fea3c3f80a1 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java @@ -0,0 +1,94 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; + +/** + * Represent the capacity in terms of physical resources like memory, disk and cpu. + * Can represent free, aggregate or total capacity of one or several nodes. + * + * @author smorgrav + */ +public class ResourceCapacity { + + private double memory; + private double cpu; + private double disk; + + ResourceCapacity() { + memory = 0; + cpu = 0; + disk = 0; + } + + ResourceCapacity(Node node) { + memory = node.flavor().getMinMainMemoryAvailableGb(); + cpu = node.flavor().getMinCpuCores(); + disk = node.flavor().getMinDiskAvailableGb(); + } + + public double getMemory() { + return memory; + } + + public double getCpu() { + return cpu; + } + + public double getDisk() { + return disk; + } + + void subtract(Node node) { + memory -= node.flavor().getMinMainMemoryAvailableGb(); + cpu -= node.flavor().getMinCpuCores(); + disk -= node.flavor().getMinDiskAvailableGb(); + } + + public static ResourceCapacity add(ResourceCapacity a, ResourceCapacity b) { + ResourceCapacity result = new ResourceCapacity(); + result.memory = a.memory + b.memory; + result.cpu = a.cpu + b.cpu; + result.disk = a.disk + b.disk; + return result; + } + + boolean hasCapacityFor(Flavor flavor) { + return memory >= flavor.getMinMainMemoryAvailableGb() && + cpu >= flavor.getMinCpuCores() && + disk >= flavor.getMinDiskAvailableGb(); + } + + int freeCapacityInFlavorEquivalence(Flavor flavor) { + if (!hasCapacityFor(flavor)) return 0; + + double memoryFactor = Math.floor(memory/flavor.getMinMainMemoryAvailableGb()); + double cpuFactor = Math.floor(cpu/flavor.getMinCpuCores()); + double diskFactor = Math.floor(disk/flavor.getMinDiskAvailableGb()); + + double aggregateFactor = Math.min(memoryFactor, cpuFactor); + aggregateFactor = Math.min(aggregateFactor, diskFactor); + + return (int)aggregateFactor; + } + + /** + * Normal compare implementation where -1 if this is less than that. + */ + int compare(ResourceCapacity that) { + if (memory > that.memory) return 1; + if (memory < that.memory) return -1; + if (disk > that.disk) return 1; + if (disk < that.disk) return -1; + if (cpu > that.cpu) return 1; + if (cpu < that.cpu) return -1; + return 0; + } + + Flavor asFlavor() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("spareflavor", cpu, memory, disk, Flavor.Type.DOCKER_CONTAINER).idealHeadroom(1); + return new Flavor(b.build().flavor(0)); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 8779038bdc2..b410b0c8ead 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -83,8 +83,12 @@ public class NodePatcher { return node.withParentHostname(asString(value)); case "ipAddresses" : return node.withIpAddresses(asStringSet(value)); + case "additionalIpAddresses" : + return node.withAdditionalIpAddresses(asStringSet(value)); case "wantToRetire" : return node.with(node.status().withWantToRetire(asBoolean(value))); + case "wantToDeprovision" : + return node.with(node.status().withWantToDeprovision(asBoolean(value))); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index a9ef14adbd9..45d0890d8cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -73,10 +73,10 @@ public class NodesApiHandler extends LoggingRequestHandler { case PATCH: return handlePATCH(request); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } - } + } catch (NotFoundException | NoSuchNodeException e) { return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - } + } catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } @@ -124,6 +124,24 @@ public class NodesApiHandler extends LoggingRequestHandler { nodeRepository.reactivate(lastElement(path), Agent.operator); return new MessageResponse("Moved " + lastElement(path) + " to active"); } + else if (path.startsWith("/nodes/v2/state/availablefornewallocations/")) { + /** + * This is a temporary "state" or rest call that we use to enable a smooth rollout of + * dynamic docker flavor allocations. Once we have switch everything we remove this + * and change the code in the nodeadmin to delete directly. + * + * Should only be called by node-admin for docker containers (the docker constraint is + * enforced in the remove method) + */ + String hostname = lastElement(path); + if (nodeRepository.dynamicAllocationEnabled()) { + nodeRepository.remove(hostname); + return new MessageResponse("Removed " + hostname); + } else { + nodeRepository.setReady(hostname); + return new MessageResponse("Moved " + hostname + " to ready"); + } + } throw new NotFoundException("Cannot put to path '" + path + "'"); } @@ -141,11 +159,11 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.equals("/nodes/v2/command/restart")) { int restartCount = nodeRepository.restart(toNodeFilter(request)).size(); return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); - } + } else if (path.equals("/nodes/v2/command/reboot")) { int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size(); return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes"); - } + } else if (path.equals("/nodes/v2/node")) { int addedNodes = addNodes(request.getData()); return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index f88f7a663f0..00bec80b758 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -171,6 +171,7 @@ class NodesResponse extends HttpResponse { object.setBool("hardwareFailure", node.status().hardwareFailure().isPresent()); node.status().hardwareFailure().ifPresent(failure -> object.setString("hardwareFailureType", toString(failure))); object.setBool("wantToRetire", node.status().wantToRetire()); + object.setBool("wantToDeprovision", node.status().wantToDeprovision()); toSlime(node.history(), object.setArray("history")); ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses")); ipAddressesToSlime(node.additionalIpAddresses(), object.setArray("additionalIpAddresses")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java index b2576c29be2..c11b906dc9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/FlavorConfigBuilder.java @@ -61,6 +61,10 @@ public class FlavorConfigBuilder { for (String flavorName : flavors) { if (flavorName.equals("docker")) flavorConfigBuilder.addFlavor(flavorName, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.DOCKER_CONTAINER); + else if (flavorName.equals("docker2")) + flavorConfigBuilder.addFlavor(flavorName, 2. /* cpu*/, 4. /* mem GB*/, 4. /*disk GB*/, Flavor.Type.DOCKER_CONTAINER); + else if (flavorName.equals("host")) + flavorConfigBuilder.addFlavor(flavorName, 7. /* cpu*/, 10. /* mem GB*/, 12. /*disk GB*/, Flavor.Type.BARE_METAL); else flavorConfigBuilder.addFlavor(flavorName, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index b29b5ee813e..538feeb042d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; +import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; @@ -62,6 +63,9 @@ public class OrchestratorMock implements Orchestrator { } @Override + public void acquirePermissionToRemove(HostName hostName) throws OrchestrationException {} + + @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchInternalErrorException, BatchHostStateChangeDeniedException, BatchHostNameNotFoundException { throw new UnsupportedOperationException("Not implemented"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index bc99667dca8..46fa5c3ce98 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -57,5 +57,30 @@ public class NodeRepositoryTest { ApplicationId.from(TenantName.from("does"), ApplicationName.from("not"), InstanceName.from("exist")); assertFalse(tester.nodeRepository().getDefaultFlavorOverride(applicationWithoutDefaultFlavor).isPresent()); } - + + @Test + public void featureToggleDynamicAllocationTest() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + assertFalse(tester.nodeRepository().dynamicAllocationEnabled()); + + tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); + assertTrue(tester.nodeRepository().dynamicAllocationEnabled()); + } + + @Test + public void only_allow_to_delete_dirty_nodes_when_dynamic_allocation_feature_enabled() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + try { + tester.addNode("id1", "host1", "default", NodeType.host); + tester.addNode("id2", "host2", "docker", NodeType.tenant); + tester.nodeRepository().setDirty("host2"); + + assertFalse(tester.nodeRepository().remove("host2")); + + tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); + assertTrue(tester.nodeRepository().remove("host2")); + } finally { + tester.curator().delete(Path.fromString("/provision/v1/dynamicDockerAllocation")); + } + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index ed6da7f82af..3c25729f1b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -52,6 +52,7 @@ public class NodeRepositoryTester { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); + b.addFlavor("docker", 1., 2., 50, Flavor.Type.DOCKER_CONTAINER).cost(1); return b.build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 6aa8520b1f5..ed4f45ece63 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -85,7 +85,7 @@ public class FailedExpirerTest { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", flavorName); NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup()); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock, (x,y) -> {}); Flavor defaultFlavor = nodeFlavors.getFlavorOrThrow("default"); List<Node> hostNodes = new ArrayList<>(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java index 3527d4ea04d..d8d1329ffbd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java @@ -34,20 +34,17 @@ public class NodeRebooterTest { assertEquals("No nodes have 2 reboots scheduled", 0, withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size()); - assertEquals("Host nodes are not rebooted", - 15, - withCurrentRebootGeneration(1L, tester.nodeRepository.getNodes(NodeType.host, Node.State.ready)).size()); maintenanceIntervals(rebooter, tester, 11); assertEquals("Reboot interval is 10x iteration interval, so most nodes are now rebooted twice", - 15, - withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(NodeType.tenant, Node.State.ready)).size()); + 30, + withCurrentRebootGeneration(3L, tester.nodeRepository.getNodes(Node.State.ready)).size()); } private void maintenanceIntervals(NodeRebooter rebooter, MaintenanceTester tester, int iterations) { for (int i = 0; i < iterations; i++) { tester.clock.advance(Duration.ofMinutes(25)); - for (int j = 0; j < 30; j++) { // multiple runs to remove effects from the probabilistic smoothing in the reboot maintainer + for (int j = 0; j < 60; j++) { // multiple runs to remove effects from the probabilistic smoothing in the reboot maintainer rebooter.maintain(); simulateReboot(tester); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java index 5e88f1c069c..16f0acc9d55 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java @@ -1,12 +1,17 @@ package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorClusters; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorClustersTest; import org.junit.Before; import org.junit.Test; import org.junit.experimental.runners.Enclosed; @@ -14,6 +19,7 @@ import org.junit.runner.RunWith; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -22,6 +28,8 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,60 +45,217 @@ import static org.mockito.Mockito.when; public class NodeRetirerTest { public static class FullNodeRepositoryTester { + private final RetirementPolicy policy = node -> node.ipAddresses().equals(Collections.singleton("::1")); + private NodeRetirerTester tester; + private NodeRetirer retirer; @Test public void testRetireUnallocatedNodes() { - NodeRetirerTester tester = new NodeRetirerTester(NodeRetirerTester.makeFlavors(5)); - RetirementPolicy policy = node -> node.ipAddresses().equals(Collections.singleton("::1")); - NodeRetirer retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, Duration.ofDays(1), new JobControl(tester.nodeRepository.database()), policy); + NodeFlavors nodeFlavors = FlavorClustersTest.makeFlavors(6); + FlavorClusters flavorClusters = new FlavorClusters(nodeFlavors.getFlavors()); + tester = new NodeRetirerTester(nodeFlavors); + retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, flavorClusters, Duration.ofDays(1), new JobControl(tester.nodeRepository.database()), policy); - tester.createReadyNodesByFlavor(5, 3, 77, 47); + tester.createReadyNodesByFlavor(7, 4, 77, 47); tester.deployApp("vespa", "calendar", 0, 3); tester.deployApp("vespa", "notes", 2, 12); tester.deployApp("sports", "results", 2, 7); tester.deployApp("search", "images", 3, 6); - Map<Flavor, Long> expected = tester.expectedCountsByFlavor(1, 3, 56, 40); - Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(tester.nodeRepository.getNodes()); - assertEquals(expected, actual); - // Not all nodes that we wanted to retire could be retired now (Not enough spare nodes) + assertSpareCountsByFlavor(2, 2, 56, 39); assertFalse(retirer.retireUnallocated()); - Map<Flavor, Long> parkedCountsByFlavor = tester.nodeRepository.getNodes(Node.State.parked).stream() - .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); - assertEquals(expected, parkedCountsByFlavor); - - expected = tester.expectedCountsByFlavor(0, -1, 0, 0); - actual = retirer.getNumberSpareReadyNodesByFlavor(tester.nodeRepository.getNodes()); - assertEquals(expected, actual); + assertParkedCountsByFlavor(2, 2, 56, 39); + assertSpareCountsByFlavor(0, 0, 0, 0); // Lets change parked nodes IP address and set it back to ready tester.nodeRepository.getNodes(Node.State.parked) .forEach(node -> { + Agent parkingAgent = node.history().event(History.Event.Type.parked).orElseThrow(RuntimeException::new).agent(); + assertEquals(Agent.NodeRetirer, parkingAgent); + assertTrue("Nodes parked by NodeRetirer should also have wantToDeprovision flag set", node.status().wantToDeprovision()); tester.nodeRepository.write(node.withIpAddresses(Collections.singleton("::2"))); tester.nodeRepository.setDirty(node.hostname()); tester.nodeRepository.setReady(node.hostname()); }); - expected = tester.expectedCountsByFlavor(1, 3, 56, 40); - actual = retirer.getNumberSpareReadyNodesByFlavor(tester.nodeRepository.getNodes()); - assertEquals(expected, actual); - // The remaining nodes we wanted to retire has been retired + assertSpareCountsByFlavor(2, 2, 56, 39); assertTrue(retirer.retireUnallocated()); - parkedCountsByFlavor = tester.nodeRepository.getNodes(Node.State.parked).stream() + assertParkedCountsByFlavor(2, 2, 2, 2); + } + + /* Creates flavors where 'replaces' graph and node counts that looks like this: + * Total nodes: 40 1 + * | 4 Total nodes: 8 + * Total nodes: 20 | | search.images nodes: 4 + * vespa.notes nodes: 3 0 | + * sports.results nodes: 6 / \ 5 Total nodes: 6 + * / \ Total nodes: 14 search.videos nodes: 2 + * Total nodes: 25 2 3 vespa.calendar nodes: 7 + */ + @Test + public void testRetireAllocatedNodes() throws InterruptedException { + NodeFlavors nodeFlavors = FlavorClustersTest.makeFlavors( + Collections.singletonList(1), // 0 -> {1} + Collections.emptyList(), // 1 -> {} + Collections.singletonList(0), // 2 -> {0} + Collections.singletonList(0), // 3 -> {0} + Collections.emptyList(), // 4 -> {} + Collections.singletonList(4)); // 5 -> {4} + FlavorClusters flavorClusters = new FlavorClusters(nodeFlavors.getFlavors()); + tester = new NodeRetirerTester(nodeFlavors); + + tester.createReadyNodesByFlavor(21, 42, 27, 15, 8, 6); + tester.deployApp("vespa", "calendar", 3, 7); + tester.deployApp("vespa", "notes", 0, 3); + tester.deployApp("sports", "results", 0, 6); + tester.deployApp("search", "images", 4, 4); + tester.deployApp("search", "videos", 5, 2); + + JobControl jobControl = new JobControl(tester.nodeRepository.database()); + retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, flavorClusters, Duration.ofDays(1), jobControl, policy); + // Update IP addresses on ready nodes so that when they are deployed to, we wont retire them + tester.nodeRepository.getNodes(Node.State.ready) + .forEach(node -> tester.nodeRepository.write(node.withIpAddresses(Collections.singleton("::2")))); + + assertSpareCountsByFlavor(10, 40, 25, 6, 2, 2); + + + retireThenAssertSpareAndParkedCounts(new long[]{8, 40, 25, 5, 1, 1}, new long[]{1, 1, 1, 1, 1}); + + // At this point we only have 1 spare node for flavors 4 & 5, 5 also replaces 4, which means that we can + // only replace 1 of either flavor-4 or flavor-5. + // search.videos (5th app) wont be replaced because search.images will get the last spare node in + // flavor-4, flavor-5 cluster because it has more active nodes + retireThenAssertSpareAndParkedCounts(new long[]{6, 40, 25, 4, 0, 1}, new long[]{2, 2, 2, 2, 1}); + + // After redeploying search.images, it ended up on a flavor-4 node, so we still have a flavor-5 spare, + // but we still wont be able to retire any nodes for search.videos as min spare for its flavor cluster is 0 + retireThenAssertSpareAndParkedCounts(new long[]{4, 40, 25, 3, 0, 1}, new long[]{3, 3, 3, 2, 1}); + + // All 3 of vespa.notes old nodes have been retired, so its parked count should stay the same + retireThenAssertSpareAndParkedCounts(new long[]{3, 40, 25, 2, 0, 1}, new long[]{4, 3, 4, 2, 1}); + + // Only vespa.calendar and sports.results remain, but their flavors (3 and 0 respectively) are in the same + // flavor cluster, because the min count for this cluster is 1, we can only retire one of them + retireThenAssertSpareAndParkedCounts(new long[]{2, 40, 25, 1, 0, 1}, new long[]{5, 3, 5, 2, 1}); + + // min flavor count for both flavor clusters is now 0, so no further change is expected + retireThenAssertSpareAndParkedCounts(new long[]{2, 40, 25, 0, 0, 1}, new long[]{6, 3, 5, 2, 1}); + retireThenAssertSpareAndParkedCounts(new long[]{2, 40, 25, 0, 0, 1}, new long[]{6, 3, 5, 2, 1}); + + tester.nodeRepository.getNodes(Node.State.parked) + .forEach(node -> assertTrue("Nodes parked by NodeRetirer should also have wantToDeprovision flag set", + node.status().wantToDeprovision())); + } + + @Test + public void testGetActiveApplicationIds() { + NodeFlavors nodeFlavors = FlavorClustersTest.makeFlavors(1); + FlavorClusters flavorClusters = new FlavorClusters(nodeFlavors.getFlavors()); + tester = new NodeRetirerTester(nodeFlavors); + retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, flavorClusters, Duration.ofDays(1), new JobControl(tester.nodeRepository.database()), policy); + + tester.createReadyNodesByFlavor(50); + ApplicationId a1 = tester.deployApp("vespa", "calendar", 0, 10); + ApplicationId a2 = tester.deployApp("vespa", "notes", 0, 12); + ApplicationId a3 = tester.deployApp("sports", "results", 0, 7); + ApplicationId a4 = tester.deployApp("search", "images", 0, 6); + + List<ApplicationId> expectedOrder = Arrays.asList(a2, a1, a3, a4); + List<ApplicationId> actualOrder = retirer.getActiveApplicationIds(tester.nodeRepository.getNodes()); + assertEquals(expectedOrder, actualOrder); + } + + @Test + public void testGetRetireableNodesForApplication() { + NodeFlavors nodeFlavors = FlavorClustersTest.makeFlavors(1); + FlavorClusters flavorClusters = new FlavorClusters(nodeFlavors.getFlavors()); + tester = new NodeRetirerTester(nodeFlavors); + retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, flavorClusters, Duration.ofDays(1), new JobControl(tester.nodeRepository.database()), policy); + + tester.createReadyNodesByFlavor(10); + tester.deployApp("vespa", "calendar", 0, 10); + + List<Node> nodes = tester.nodeRepository.getNodes(); + Set<String> actual = retirer.getRetireableNodesForApplication(nodes).stream().map(Node::hostname).collect(Collectors.toSet()); + Set<String> expected = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + assertEquals(expected, actual); + + Node nodeWantToRetire = tester.nodeRepository.getNode("host3.test.yahoo.com").orElseThrow(RuntimeException::new); + tester.nodeRepository.write(nodeWantToRetire.with(nodeWantToRetire.status().withWantToRetire(true))); + Node nodeToFail = tester.nodeRepository.getNode("host5.test.yahoo.com").orElseThrow(RuntimeException::new); + tester.nodeRepository.fail(nodeToFail.hostname(), Agent.system, "Failed for unit testing"); + Node nodeToUpdate = tester.nodeRepository.getNode("host8.test.yahoo.com").orElseThrow(RuntimeException::new); + tester.nodeRepository.write(nodeToUpdate.withIpAddresses(Collections.singleton("::2"))); + + nodes = tester.nodeRepository.getNodes(); + Set<String> excluded = Stream.of(nodeWantToRetire, nodeToFail, nodeToUpdate).map(Node::hostname).collect(Collectors.toSet()); + Set<String> actualAfterUpdates = retirer.getRetireableNodesForApplication(nodes).stream().map(Node::hostname).collect(Collectors.toSet()); + Set<String> expectedAfterUpdates = nodes.stream().map(Node::hostname).filter(node -> !excluded.contains(node)).collect(Collectors.toSet()); + assertEquals(expectedAfterUpdates, actualAfterUpdates); + } + + @Test + public void testGetNumberNodesAllowToRetireForApplication() { + NodeFlavors nodeFlavors = FlavorClustersTest.makeFlavors(1); + FlavorClusters flavorClusters = new FlavorClusters(nodeFlavors.getFlavors()); + tester = new NodeRetirerTester(nodeFlavors); + retirer = new NodeRetirer(tester.nodeRepository, NodeRetirerTester.zone, flavorClusters, Duration.ofDays(1), new JobControl(tester.nodeRepository.database()), policy); + + tester.createReadyNodesByFlavor(10); + tester.deployApp("vespa", "calendar", 0, 10); + + long actualAllActive = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(), 2); + assertEquals(2, actualAllActive); + + // Lets put 3 random nodes in wantToRetire + List<Node> nodesToRetire = tester.nodeRepository.getNodes().stream().limit(3).collect(Collectors.toList()); + nodesToRetire.forEach(node -> tester.nodeRepository.write(node.with(node.status().withWantToRetire(true)))); + long actualOneWantToRetire = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(), 2); + assertEquals(0, actualOneWantToRetire); + + // Now 2 of those finish retiring and go to parked + nodesToRetire.stream().limit(2).forEach(node -> + tester.nodeRepository.park(node.hostname(), Agent.system, "Parked for unit testing")); + long actualOneRetired = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(), 2); + assertEquals(1, actualOneRetired); + } + + private void assertSpareCountsByFlavor(long... nums) { + Map<Flavor, Long> expectedSpareCountsByFlavor = tester.expectedCountsByFlavor(nums); + Map<Flavor, Long> actualSpaceCountsByFlavor = retirer.getNumberSpareReadyNodesByFlavor(tester.nodeRepository.getNodes()); + assertEquals(expectedSpareCountsByFlavor, actualSpaceCountsByFlavor); + } + + private void assertParkedCountsByFlavor(long... nums) { + Map<Flavor, Long> expected = tester.expectedCountsByFlavor(nums); + Map<Flavor, Long> actual = tester.nodeRepository.getNodes(Node.State.parked).stream() .collect(Collectors.groupingBy(Node::flavor, Collectors.counting())); - expected = tester.expectedCountsByFlavor(1, -1, 2, 1); - assertEquals(expected, parkedCountsByFlavor); + assertEquals(expected, actual); + } + + private void assertParkedCountsByApplication(long... nums) { + Map<ApplicationId, Long> expected = tester.expectedCountsByApplication(nums); + Map<ApplicationId, Long> actual = tester.nodeRepository.getNodes(Node.State.parked).stream() + .collect(Collectors.groupingBy(node -> node.allocation().get().owner(), Collectors.counting())); + assertEquals(expected, actual); } - } + private void retireThenAssertSpareAndParkedCounts(long[] spareCountsByFlavor, long[] parkedCountsByApp) { + retirer.retireAllocated(); + tester.iterateMaintainers(); + assertSpareCountsByFlavor(spareCountsByFlavor); + assertParkedCountsByApplication(parkedCountsByApp); + } + } /** * For testing methods that require minimal node repository and flavor setup */ public static class HelperMethodsTester { - private final List<Flavor> flavors = NodeRetirerTester.makeFlavors(5).getFlavors(); + private final List<Flavor> flavors = FlavorClustersTest.makeFlavors(5).getFlavors(); private final List<Node> nodes = new ArrayList<>(); private final NodeRetirer retirer = mock(NodeRetirer.class); @@ -98,7 +263,7 @@ public class NodeRetirerTest { public void testGetNumberSpareNodesWithNoActiveNodes() { addNodesByFlavor(Node.State.ready, 5, 3, 77); - Map<Flavor, Long> expected = expectedCountsByFlavor(5, 3, 77); + Map<Flavor, Long> expected = expectedCountsByFlavor(3, 1, 75); Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(nodes); assertEquals(expected, actual); } @@ -108,7 +273,7 @@ public class NodeRetirerTest { addNodesByFlavor(Node.State.ready, 5, 3, 77, 47); addNodesByFlavor(Node.State.active, 0, 10, 2, 230, 137); - Map<Flavor, Long> expected = expectedCountsByFlavor(5, 2, 76, 24); + Map<Flavor, Long> expected = expectedCountsByFlavor(3, 1, 75, 45); Map<Flavor, Long> actual = retirer.getNumberSpareReadyNodesByFlavor(nodes); assertEquals(expected, actual); } @@ -166,13 +331,30 @@ public class NodeRetirerTest { when(retirer.getNumSpareNodes(any(Long.class), any(Long.class))).thenCallRealMethod(); assertEquals(retirer.getNumSpareNodes(0, 0), 0L); - assertEquals(retirer.getNumSpareNodes(0, 1), 1L); - assertEquals(retirer.getNumSpareNodes(0, 100), 100L); + assertEquals(retirer.getNumSpareNodes(0, 1), 0L); + assertEquals(retirer.getNumSpareNodes(0, 100), 98L); assertEquals(retirer.getNumSpareNodes(1, 0), 0L); assertEquals(retirer.getNumSpareNodes(1, 1), 0L); - assertEquals(retirer.getNumSpareNodes(1, 2), 1L); - assertEquals(retirer.getNumSpareNodes(43, 23), 18L); + assertEquals(retirer.getNumSpareNodes(1, 2), 0L); + assertEquals(retirer.getNumSpareNodes(43, 23), 21L); + } + + @Test + public void testGetMinAmongstKeys() { + when(retirer.getMinAmongstKeys(any(), any())).thenCallRealMethod(); + + Map<String, Integer> map = createMapWith(4, 10, 43, 23, 7, 53, 2, 12, 42, 10); + Set<String> keys = createKeySetWith(1, 3, 4, 5, 7, 9); + assertEquals("4", retirer.getMinAmongstKeys(map, keys)); // Smallest value is 7, which is index 4 + } + + private Map<String, Integer> createMapWith(int... values) { + return IntStream.range(0, values.length).boxed().collect(Collectors.toMap(String::valueOf, i -> values[i])); + } + + private Set<String> createKeySetWith(int... keys) { + return Arrays.stream(keys).boxed().map(String::valueOf).collect(Collectors.toSet()); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java index 635e7ac7d10..c6412d7c28f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -21,19 +20,20 @@ import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; -import com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub; -import com.yahoo.vespa.orchestrator.Orchestrator; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** @@ -45,12 +45,16 @@ public class NodeRetirerTester { // Components with state public final ManualClock clock; public final NodeRepository nodeRepository; - public ServiceMonitorStub serviceMonitor; - public MockDeployer deployer; - private final Orchestrator orchestrator; private final NodeRepositoryProvisioner provisioner; private final Curator curator; private final List<Flavor> flavors; + + // Use LinkedHashMap to keep order in which applications were deployed + private final Map<ApplicationId, MockDeployer.ApplicationContext> apps = new LinkedHashMap<>(); + + private PeriodicApplicationMaintainer applicationMaintainer; + private RetiredExpirer retiredExpirer; + private InactiveExpirer inactiveExpirer; private int nextNodeId = 0; public NodeRetirerTester(NodeFlavors nodeFlavors) { @@ -58,18 +62,7 @@ public class NodeRetirerTester { curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup()); provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); - orchestrator = new OrchestratorMock(); - deployer = new MockDeployer(provisioner, Collections.emptyMap()); - serviceMonitor = new ServiceMonitorStub(Collections.emptyMap(), nodeRepository); - flavors = nodeFlavors.getFlavors(); - } - - public void suspend(ApplicationId app) { - try { - orchestrator.suspend(app); - } catch (Exception e) { - throw new RuntimeException(e); - } + flavors = nodeFlavors.getFlavors().stream().sorted(Comparator.comparing(Flavor::name)).collect(Collectors.toList()); } public void createReadyNodesByFlavor(int... nums) { @@ -88,14 +81,36 @@ public class NodeRetirerTester { nodeRepository.setReady(nodes); } - public void deployApp(String tenantName, String applicationName, int flavorId, int numNodes) { + public ApplicationId deployApp(String tenantName, String applicationName, int flavorId, int numNodes) { Flavor flavor = flavors.get(flavorId); ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.99")); Capacity capacity = Capacity.fromNodeCount(numNodes, flavor.name()); + apps.put(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, capacity, 1)); activate(applicationId, cluster, capacity); + return applicationId; + } + + public void iterateMaintainers() { + if (applicationMaintainer == null) { + MockDeployer deployer = new MockDeployer(provisioner, apps); + JobControl jobControl = new JobControl(nodeRepository.database()); + applicationMaintainer = new PeriodicApplicationMaintainerTest.TestablePeriodicApplicationMaintainer( + deployer, nodeRepository, Duration.ofMinutes(10), Optional.empty()); + retiredExpirer = new RetiredExpirer(nodeRepository, deployer, clock, Duration.ofMinutes(10), jobControl); + inactiveExpirer = new InactiveExpirer(nodeRepository, clock, Duration.ofMinutes(10), jobControl); + + } + + applicationMaintainer.maintain(); + + clock.advance(Duration.ofMinutes(11)); + retiredExpirer.maintain(); + + clock.advance(Duration.ofMinutes(11)); + inactiveExpirer.maintain(); } private void activate(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { @@ -115,11 +130,14 @@ public class NodeRetirerTester { return countsByFlavor; } - public static NodeFlavors makeFlavors(int numFlavors) { - FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); - for (int i = 0; i < numFlavors; i++) { - flavorConfigBuilder.addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); + public Map<ApplicationId, Long> expectedCountsByApplication(long... nums) { + Map<ApplicationId, Long> countsByApplicationId = new HashMap<>(); + Iterator<ApplicationId> iterator = apps.keySet().iterator(); + for (int i = 0; iterator.hasNext(); i++) { + if (nums[i] < 0) continue; + ApplicationId applicationId = iterator.next(); + countsByApplicationId.put(applicationId, nums[i]); } - return new NodeFlavors(flavorConfigBuilder.build()); + return countsByApplicationId; } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index cd4a34cf6ea..c1504ec188d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -204,7 +204,7 @@ public class PeriodicApplicationMaintainerTest { } - private static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer { + public static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer { private Optional<List<Node>> overriddenNodesNeedingMaintenance; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index f78dc031b0d..1b9371d44ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -40,7 +40,7 @@ public class ReservationExpirerTest { NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup()); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock, (x,y) -> {}); List<Node> nodes = new ArrayList<>(2); nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index f034490b3f7..9bfeccb9a5d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; @@ -20,11 +21,12 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import com.yahoo.vespa.orchestrator.OrchestrationException; +import com.yahoo.vespa.orchestrator.Orchestrator; import org.junit.Test; import java.time.Duration; @@ -35,6 +37,11 @@ import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * @author bratseth @@ -58,7 +65,7 @@ public class RetiredExpirerTest { ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); // Allocate content cluster of sizes 7 -> 2 -> 3: - // Should end up with 3 nodes in the cluster (one previously retired), and 3 retired + // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42")); int wantedNodes; activate(applicationId, cluster, wantedNodes=7, 1, provisioner); @@ -117,6 +124,64 @@ public class RetiredExpirerTest { assertFalse(node.allocation().get().membership().retired()); } + @Test + public void ensure_early_inactivation() throws OrchestrationException { + ManualClock clock = new ManualClock(); + Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + new MockNameResolver().mockAnyLookup()); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); + + createReadyNodes(7, nodeRepository, nodeFlavors); + createHostNodes(4, nodeRepository, nodeFlavors); + + ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); + + // Allocate content cluster of sizes 7 -> 2 -> 3: + // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42")); + int wantedNodes; + activate(applicationId, cluster, wantedNodes=7, 1, provisioner); + activate(applicationId, cluster, wantedNodes=2, 1, provisioner); + activate(applicationId, cluster, wantedNodes=3, 1, provisioner); + assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + + // Cause inactivation of retired nodes + clock.advance(Duration.ofHours(30)); // Retire period spent + MockDeployer deployer = + new MockDeployer(provisioner, + Collections.singletonMap( + applicationId, + new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromNodeCount(wantedNodes, Optional.of("default")), 1))); + + Orchestrator orchestrator = mock(Orchestrator.class); + // Allow the 1st and 3rd retired nodes permission to inactivate + doNothing() + .doThrow(new OrchestrationException("Permission not granted 1")) + .doNothing() + .doThrow(new OrchestrationException("Permission not granted 2")) + .when(orchestrator).acquirePermissionToRemove(any()); + + new RetiredEarlyExpirer( + nodeRepository, + zone, + Duration.ofDays(30), + new JobControl(nodeRepository.database()), + deployer, + orchestrator).run(); + assertEquals(5, nodeRepository.getNodes(applicationId, Node.State.active).size()); + assertEquals(2, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); + assertEquals(1, deployer.redeployments); + + verify(orchestrator, times(4)).acquirePermissionToRemove(any()); + + // inactivated nodes are not retired + for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive)) + assertFalse(node.allocation().get().membership().retired()); + } + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) { List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(nodes), groups, null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java index fdfd4bdd8bf..690d078cdb6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/MetricsReporterTest.java @@ -1,6 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.monitoring; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; @@ -8,25 +12,30 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.maintenance.JobControl; import com.yahoo.vespa.hosted.provision.maintenance.MetricsReporter; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Clock; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CountDownLatch; +import java.util.Set; import static org.junit.Assert.assertEquals; /** * @author oyving + * @author smorgrav */ public class MetricsReporterTest { @@ -38,7 +47,7 @@ public class MetricsReporterTest { new MockNameResolver().mockAnyLookup()); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(Collections.singletonList(node)); - Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host); + Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); nodeRepository.addNodes(Collections.singletonList(hostNode)); Map<String, Number> expectedMetrics = new HashMap<>(); @@ -50,6 +59,12 @@ public class MetricsReporterTest { expectedMetrics.put("hostedVespa.inactiveHosts", 0); expectedMetrics.put("hostedVespa.dirtyHosts", 0); expectedMetrics.put("hostedVespa.failedHosts", 0); + expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); + expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); + expectedMetrics.put("hostedVespa.docker.totalCapacityCpu", 0.0); + expectedMetrics.put("hostedVespa.docker.freeCapacityDisk", 0.0); + expectedMetrics.put("hostedVespa.docker.freeCapacityMem", 0.0); + expectedMetrics.put("hostedVespa.docker.freeCapacityCpu", 0.0); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, metric, Duration.ofMinutes(1), new JobControl(nodeRepository.database())); @@ -58,28 +73,138 @@ public class MetricsReporterTest { assertEquals(expectedMetrics, metric.values); } + @Test + public void docker_metrics() throws InterruptedException { + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); + Curator curator = new MockCurator(); + NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), + new MockNameResolver().mockAnyLookup()); + + // Allow 4 containers + Set<String> additionalIps = new HashSet<>(); + additionalIps.add("::2"); + additionalIps.add("::3"); + additionalIps.add("::4"); + additionalIps.add("::5"); + + Node dockerHost = Node.create("openStackId1", Collections.singleton("::1"), additionalIps, "dockerHost", Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + nodeRepository.addNodes(Collections.singletonList(dockerHost)); + nodeRepository.setDirty("dockerHost"); + nodeRepository.setReady("dockerHost"); + + Node container1 = Node.createDockerNode("openStackId1:1", Collections.singleton("::2"), Collections.emptySet(), "container1", Optional.of("dockerHost"), nodeFlavors.getFlavorOrThrow("docker"), NodeType.tenant); + container1 = container1.with(allocation(Optional.of("app1")).get()); + nodeRepository.addDockerNodes(Collections.singletonList(container1)); + + Node container2 = Node.createDockerNode("openStackId1:2", Collections.singleton("::3"), Collections.emptySet(), "container2", Optional.of("dockerHost"), nodeFlavors.getFlavorOrThrow("docker2"), NodeType.tenant); + container2 = container2.with(allocation(Optional.of("app2")).get()); + nodeRepository.addDockerNodes(Collections.singletonList(container2)); + + TestMetric metric = new TestMetric(); + MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, metric, Duration.ofMinutes(1), new JobControl(nodeRepository.database())); + metricsReporter.maintain(); + + assertEquals(0, metric.values.get("hostedVespa.readyHosts")); /** Only tenants counts **/ + assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); + + assertEquals(12.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); + assertEquals(10.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); + assertEquals(7.0, metric.values.get("hostedVespa.docker.totalCapacityCpu")); + + assertEquals(6.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); + assertEquals(3.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); + assertEquals(4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); + + assertContext(metric, "hostedVespa.docker.freeCapacityFlavor", 1, 0); + assertContext(metric, "hostedVespa.docker.idealHeadroomFlavor", 0, 0); + assertContext(metric, "hostedVespa.docker.hostsAvailableFlavor", 1l, 0l); + } + + private void assertContext(TestMetric metric, String key, Number dockerValue, Number docker2Value) { + List<Metric.Context> freeCapacityFlavor = metric.context.get(key); + assertEquals(freeCapacityFlavor.size(), 2); + + // Get the value for the two flavors + TestMetric.TestContext contextFlavorDocker = (TestMetric.TestContext)freeCapacityFlavor.get(0); + TestMetric.TestContext contextFlavorDocker2 = (TestMetric.TestContext)freeCapacityFlavor.get(1); + if (!contextFlavorDocker.properties.containsValue("docker")) { + TestMetric.TestContext temp = contextFlavorDocker; + contextFlavorDocker = contextFlavorDocker2; + contextFlavorDocker2 = temp; + } + + assertEquals(dockerValue, contextFlavorDocker.value); + assertEquals(docker2Value, contextFlavorDocker2.value); + + } + + private ApplicationId app(String tenant) { + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName("test") + .instanceName("default").build(); + } + + private Optional<Allocation> allocation(Optional<String> tenant) { + if (tenant.isPresent()) { + Allocation allocation = new Allocation(app(tenant.get()), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + return Optional.of(allocation); + } + return Optional.empty(); + } + private static class TestMetric implements Metric { public Map<String, Number> values = new HashMap<>(); - public Map<String, Context> context = new HashMap<>(); + public Map<String, List<Context>> context = new HashMap<>(); @Override public void set(String key, Number val, Context ctx) { values.put(key, val); - context.put(key, ctx); + if (ctx != null) { + //Create one context pr value added - copy the context to not have side effects + TestContext kontekst = (TestContext)createContext(((TestContext) ctx).properties); + if (!context.containsKey(key)) { + context.put(key, new ArrayList<>()); + } + kontekst.setValue(val); + context.get(key).add(kontekst); + } } @Override public void add(String key, Number val, Context ctx) { values.put(key, val); - context.put(key, ctx); + if (ctx != null) { + //Create one context pr value added - copy the context to not have side effects + TestContext kontekst = (TestContext)createContext(((TestContext) ctx).properties); + if (!context.containsKey(key)) { + context.put(key, new ArrayList<>()); + } + kontekst.setValue(val); + context.get(key).add(kontekst); + } } @Override public Context createContext(Map<String, ?> properties) { - return null; + return new TestContext(properties); } - } + /** + * Context where the propertymap is not shared - but unique to each value. + */ + private static class TestContext implements Context{ + Number value; + Map<String, ?> properties; + + public TestContext(Map<String, ?> properties) { + this.properties = properties; + } + public void setValue(Number value) { + this.value = value; + } + } + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 4fbd5bd1a66..d2c3e60b420 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -131,7 +131,8 @@ public class SerializationTest { " \"serviceId\" : \"content/myId/0\",\n" + " \"restartGeneration\" : 3,\n" + " \"currentRestartGeneration\" : 4,\n" + - " \"removable\" : true\n" + + " \"removable\" : true,\n" + + " \"wantedVespaVersion\": \"6.42.2\"\n" + " },\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + @@ -188,7 +189,8 @@ public class SerializationTest { " \"applicationId\": \"ugc-assimilate\",\n" + " \"instanceId\": \"default\",\n" + " \"serviceId\": \"container/ugccloud-container/0/0\",\n" + - " \"restartGeneration\": 0\n" + + " \"restartGeneration\": 0,\n" + + " \"wantedVespaVersion\": \"6.42.2\"\n" + " }\n" + "}\n").getBytes()); assertEquals(0, node.history().events().size()); @@ -323,24 +325,21 @@ public class SerializationTest { } @Test - // TODO: Remove after April 2017 - public void vespa_version_serialization() throws Exception { - String nodeWithDockerImage = + public void want_to_deprovision_defaults_to_false() { + String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + " \"flavor\" : \"large\",\n" + " \"openStackId\" : \"myId\",\n" + " \"hostname\" : \"myHostname\",\n" + - " \"ipAddresses\" : [\"127.0.0.1\"],\n" + - " \"instance\": {\n" + - " \"serviceId\": \"content/myId/0\",\n" + - " \"dockerImage\": \"docker-registry.some.domain:4443/vespa/ci:6.42.1\"\n" + - " }\n" + + " \"ipAddresses\" : [\"127.0.0.1\"]\n" + "}"; - Node node = nodeSerializer.fromJson(State.active, Utf8.toBytes(nodeWithDockerImage)); - assertEquals("6.42.1", node.allocation().get().membership().cluster().vespaVersion().toString()); - assertEquals("docker-registry.ops.yahoo.com:4443/vespa/ci:6.42.1", node.allocation().get().membership().cluster().dockerImage()); + Node node = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData)); + assertFalse(node.status().wantToDeprovision()); + } + @Test + public void vespa_version_serialization() throws Exception { String nodeWithWantedVespaVersion = "{\n" + " \"type\" : \"tenant\",\n" + @@ -353,7 +352,7 @@ public class SerializationTest { " \"wantedVespaVersion\": \"6.42.2\"\n" + " }\n" + "}"; - node = nodeSerializer.fromJson(State.active, Utf8.toBytes(nodeWithWantedVespaVersion)); + Node node = nodeSerializer.fromJson(State.active, Utf8.toBytes(nodeWithWantedVespaVersion)); assertEquals("6.42.2", node.allocation().get().membership().cluster().vespaVersion().toString()); assertEquals("docker-registry.ops.yahoo.com:4443/vespa/ci:6.42.2", node.allocation().get().membership().cluster().dockerImage()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 6ba2c40186d..59601832e59 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester.createConfig; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java new file mode 100644 index 00000000000..7a24e4eaeaf --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -0,0 +1,154 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.lang.MutableInteger; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; + +import javax.swing.JFrame; +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + + +/** + * Graphically run allocation procedure to ease manual comprehension. + * + * Extremely useful when trying to understand test cases and build corner cases. + */ +public class AllocationSimulator { + + private AllocationVisualizer visualizer; + private NodeList nodes = new NodeList(new ArrayList<>()); + private NodeFlavors flavors; + + private AllocationSimulator(AllocationVisualizer visualizer) { + this.visualizer = visualizer; + + // + // Setup flavors + // + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 8., 8., 8, Flavor.Type.BARE_METAL).idealHeadroom(1); + b.addFlavor("host-small", 5., 5., 5, Flavor.Type.BARE_METAL).idealHeadroom(2); + b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + flavors = new NodeFlavors(b.build()); + + // + // Initiate nodes in system + // + List<Node> initialNodes = new ArrayList<>(); + initialNodes.add(host("host1", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host2", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host3", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host4", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host5", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host6", flavors.getFlavorOrThrow("host-large"))); + initialNodes.add(host("host7", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host8", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host9", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(host("host10", flavors.getFlavorOrThrow("host-small"))); + initialNodes.add(node("node1", flavors.getFlavorOrThrow("d-2"), Optional.of("host1"), Optional.of("test"))); + nodes = new NodeList(initialNodes); + + visualizer.addStep(nodes.asList(), "Initial state", ""); + } + + /* ------------ Create node and flavor methods ----------------*/ + + private Node host(String hostname, Flavor flavor) { + return node(hostname, flavor, Optional.empty(), Optional.empty()); + } + + private Node node(String hostname, Flavor flavor, Optional<String> parent, Optional<String> tenant) { + return new Node("fake", Collections.singleton("127.0.0.1"), + parent.isPresent() ? Collections.emptySet() : getAdditionalIP(), hostname, parent, flavor, Status.initial(), + parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant), History.empty(), parent.isPresent() ? NodeType.tenant : NodeType.host); + } + + private Set<String> getAdditionalIP() { + Set<String> h = new HashSet<String>(); + Collections.addAll(h, "::1", "::2", "::3", "::4", "::5", "::6", "::7", "::8"); + return h; + } + + private Optional<Allocation> allocation(Optional<String> tenant) { + if (tenant.isPresent()) { + Allocation allocation = new Allocation(app(tenant.get()), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + return Optional.of(allocation); + } + return Optional.empty(); + } + + private ApplicationId app(String tenant) { + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName("test") + .instanceName("default").build(); + } + + private ClusterSpec cluster() { + return ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), ClusterSpec.Group.from(1), Version.fromString("6.41")); + } + + /* ------------ Methods to add events to the system ----------------*/ + + public void addCluster(String task, int count, Flavor flavor, String id) { + NodeSpec.CountNodeSpec nodeSpec = new NodeSpec.CountNodeSpec(count, flavor); + NodeAllocation allocation = new NodeAllocation(app(id), cluster(), nodeSpec, new MutableInteger(0), Clock.systemUTC()); + + List<Node> accepted = DockerAllocator.allocateNewDockerNodes(allocation, + nodeSpec, + new ArrayList<>(nodes.asList()), + new ArrayList<>(nodes.asList()), + flavors, + flavor, + 2, + (nodes, message)-> visualizer.addStep(nodes, id, message)); + + accepted.addAll(nodes.asList()); + nodes = new NodeList(accepted); + } + + + public static void main(String[] args) { + + AllocationVisualizer visualisator = new AllocationVisualizer(); + + javax.swing.SwingUtilities.invokeLater(() -> { + JFrame frame = new JFrame("Allocation Simulator"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setContentPane(visualisator); + frame.pack(); + frame.setVisible(true); + }); + + AllocationSimulator simulator = new AllocationSimulator(visualisator); + simulator.addCluster("App1 : 3 * d-1 nodes", 3, simulator.flavors.getFlavorOrThrow("d-1"), "App1"); + simulator.addCluster("App2 : 2 * d-2 nodes", 2, simulator.flavors.getFlavorOrThrow("d-2"), "App2"); + simulator.addCluster("App3 : 3 * d-2 nodes", 3, simulator.flavors.getFlavorOrThrow("d-2"), "App3"); + simulator.addCluster("App4 : 3 * d-3 nodes", 3, simulator.flavors.getFlavorOrThrow("d-3"), "App4"); + simulator.addCluster("App5 : 3 * d-3 nodes", 3, simulator.flavors.getFlavorOrThrow("d-3"), "App5"); + simulator.addCluster("App6 : 5 * d-2 nodes", 5, simulator.flavors.getFlavorOrThrow("d-2"), "App6"); + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java new file mode 100644 index 00000000000..80e9670546d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSnapshot.java @@ -0,0 +1,18 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.vespa.hosted.provision.NodeList; + +/** + * @author smorgrav + */ +public class AllocationSnapshot { + NodeList nodes; + String message; + String task; + + AllocationSnapshot(NodeList nodes, String task, String message) { + this.nodes = nodes; + this.message = message; + this.task = task; + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java new file mode 100644 index 00000000000..ba9a7f2924f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -0,0 +1,143 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; + +import javax.swing.JButton; +import javax.swing.JFrame; +import javax.swing.JPanel; +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.Graphics; +import java.util.ArrayList; +import java.util.List; + +/** + * @author smorgrav + */ +public class AllocationVisualizer extends JPanel { + // Container box's width and height + private static final int BOX_WIDTH = 1024; + private static final int BOX_HEIGHT = 480; + + // node properties + private int nodeWidth = BOX_WIDTH / 15; + private int nodeHeight = nodeWidth / 2; + private int nodeSpacing = nodeWidth / 3; + + private final List<AllocationSnapshot> steps; + int step = 0; + + public AllocationVisualizer() { + this(new ArrayList<>()); + } + + public AllocationVisualizer(List<AllocationSnapshot> steps) { + this.steps = steps; + this.setPreferredSize(new Dimension(BOX_WIDTH, BOX_HEIGHT)); + + JButton back = new JButton("Back"); + back.addActionListener(e -> { + if (step > 0) step -= 1; + repaint(); + }); + JButton forward = new JButton("Forward"); + forward.addActionListener(e -> { + if (step < steps.size() - 1) step += 1; + repaint(); + }); + this.add(back); + this.add(forward); + } + + + public void addStep(List<Node> nodes, String task, String message) { + steps.add(new AllocationSnapshot(new NodeList(nodes), task, message)); + } + + @Override + public void paintComponent(Graphics g) { + super.paintComponent(g); + System.out.println("PAINTING"); + if (steps.size() == 0) return; + + int nodeX = 40; + int nodeY = BOX_HEIGHT - 20; //Start at the bottom + + // Draw the box + g.setColor(Color.WHITE); + g.fillRect(0, 0, BOX_WIDTH, BOX_HEIGHT); + + // Find number of docker hosts (to calculate start, and width of each) + // Draw the docker hosts - and color each container according to application + AllocationSnapshot simStep = steps.get(step); + NodeList hosts = simStep.nodes.nodeType(NodeType.host); + for (Node host : hosts.asList()) { + + // Paint the host + paintNode(host, g, nodeX, nodeY, true); + + // Paint containers + NodeList containers = simStep.nodes.childNodes(host); + for (Node container : containers.asList()) { + nodeY = paintNode(container, g, nodeX, nodeY, false); + } + + // Next host + nodeX += nodeWidth + nodeSpacing; + nodeY = BOX_HEIGHT - 20; + } + + // Display messages + g.setColor(Color.BLACK); + g.setFont(new Font("Courier New", Font.BOLD, 15)); + g.drawString(simStep.task, 20, 30); + g.drawString(simStep.message, 20, 50); + } + + private int paintNode(Node node, Graphics g, int x, int y, boolean isHost) { + + if (isHost) { + g.setColor(Color.GRAY); + for (int i = 0; i < node.flavor().getMinMainMemoryAvailableGb(); i++) { + g.fillRect(x, y - nodeHeight, nodeWidth, nodeHeight); + y = y - (nodeHeight + 2); + } + } else { + g.setColor(Color.YELLOW); + int multi = (int) node.flavor().getMinMainMemoryAvailableGb(); + int height = multi * nodeHeight + ((multi - 1) * 2); + g.fillRect(x, y - height, nodeWidth, height); + + // Write tenant name in allocation + String tenantName = node.allocation().get().owner().tenant().value(); + g.setColor(Color.BLACK); + g.setFont(new Font("Courier New", Font.PLAIN, 12)); + g.drawString(tenantName, x + nodeWidth / 2 - 20, y - height / 2); + + y = y - height - 2; + } + return y; + } + + public static void visualize(List<AllocationSnapshot> snaps) { + AllocationVisualizer visualisator = new AllocationVisualizer(snaps); + javax.swing.SwingUtilities.invokeLater(() -> { + JFrame frame = new JFrame("Allocation Simulator"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setContentPane(visualisator); + frame.pack(); + frame.setVisible(true); + }); + + while(true) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java new file mode 100644 index 00000000000..464dd83211a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -0,0 +1,156 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author smorgrav + */ +public class DockerHostCapacityTest { + + private DockerHostCapacity capacity; + private List<Node> nodes; + private Node host1, host2, host3; + Node nodeA, nodeB, nodeC, nodeD, nodeE; + Flavor flavorDocker, flavorDocker2; + + @Before + public void setup() { + // Create flavors + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); + flavorDocker = nodeFlavors.getFlavorOrThrow("docker"); + flavorDocker2 = nodeFlavors.getFlavorOrThrow("docker2"); + + // Create three docker hosts + host1 = Node.create("host1", Collections.singleton("::1"), generateIPs(2, 4), "host1", Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host2 = Node.create("host2", Collections.singleton("::11"), generateIPs(12, 3), "host2", Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host3 = Node.create("host3", Collections.singleton("::21"), generateIPs(22, 1), "host3", Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + + // Add two containers to host1 + nodeA = Node.create("nodeA", Collections.singleton("::2"), Collections.emptySet(), "nodeA", Optional.of("host1"), flavorDocker, NodeType.tenant); + nodeB = Node.create("nodeB", Collections.singleton("::3"), Collections.emptySet(), "nodeB", Optional.of("host1"), flavorDocker, NodeType.tenant); + + // Add two containers to host 2 (same as host 1) + nodeC = Node.create("nodeC", Collections.singleton("::12"), Collections.emptySet(), "nodeC", Optional.of("host2"), flavorDocker, NodeType.tenant); + nodeD = Node.create("nodeD", Collections.singleton("::13"), Collections.emptySet(), "nodeD", Optional.of("host2"), flavorDocker, NodeType.tenant); + + // Add a larger container to host3 + nodeE = Node.create("nodeE", Collections.singleton("::22"), Collections.emptySet(), "nodeE", Optional.of("host3"), flavorDocker2, NodeType.tenant); + + // init docker host capacity + nodes = new ArrayList<>(); + Collections.addAll(nodes, host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE); + capacity = new DockerHostCapacity(nodes); + } + + @Test + public void compare_used_to_sort_in_decending_order() { + assertEquals(host1, nodes.get(0)); //Make sure it is unsorted here + Collections.sort(nodes, capacity::compare); + assertEquals(host3, nodes.get(0)); + assertEquals(host1, nodes.get(1)); + assertEquals(host2, nodes.get(2)); + + // Replace a node for host 2 with a headroom node - host2 should then be prioritized + Allocation allocation = new Allocation(app(DockerHostCapacity.HEADROOM_TENANT), ClusterMembership.from("container/id1/3", new Version()), Generation.inital(), false); + Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host2"), flavorDocker, NodeType.tenant); + Node nodeFWithAllocation = nodeF.with(allocation); + nodes.add(nodeFWithAllocation); + nodes.remove(nodeC); + capacity = new DockerHostCapacity(nodes); + Collections.sort(nodes, capacity::compare); + assertEquals(host3, nodes.get(0)); + assertEquals(host2, nodes.get(1)); + assertEquals(host1, nodes.get(2)); + } + + @Test + public void hasCapacity() { + assertTrue(capacity.hasCapacity(host1, flavorDocker)); + assertTrue(capacity.hasCapacity(host1, flavorDocker2)); + assertTrue(capacity.hasCapacity(host2, flavorDocker)); + assertTrue(capacity.hasCapacity(host2, flavorDocker2)); + assertFalse(capacity.hasCapacity(host3, flavorDocker)); // No ip available + assertFalse(capacity.hasCapacity(host3, flavorDocker2)); // No ip available + + // Add a new node to host1 to deplete the memory resource + Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), + "nodeF", Optional.of("host1"), flavorDocker, NodeType.tenant); + nodes.add(nodeF); + capacity = new DockerHostCapacity(nodes); + assertFalse(capacity.hasCapacity(host1, flavorDocker)); + assertFalse(capacity.hasCapacity(host1, flavorDocker2)); + } + + @Test + public void freeIPs() { + assertEquals(2, capacity.freeIPs(host1)); + assertEquals(1, capacity.freeIPs(host2)); + assertEquals(0, capacity.freeIPs(host3)); + } + + @Test + public void getCapacityTotal() { + ResourceCapacity total = capacity.getCapacityTotal(); + assertEquals(21.0, total.getCpu(), 0.1); + assertEquals(30.0, total.getMemory(), 0.1); + assertEquals(36.0, total.getDisk(), 0.1); + } + + @Test + public void getFreeCapacityTotal() { + ResourceCapacity totalFree = capacity.getFreeCapacityTotal(); + assertEquals(15.0, totalFree.getCpu(), 0.1); + assertEquals(14.0, totalFree.getMemory(), 0.1); + assertEquals(24.0, totalFree.getDisk(), 0.1); + } + + @Test + public void freeCapacityInFlavorEquivalence() { + assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker)); + assertEquals(2, capacity.freeCapacityInFlavorEquivalence(flavorDocker2)); + } + + @Test + public void getNofHostsAvailableFor() { + assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker)); + assertEquals(2, capacity.getNofHostsAvailableFor(flavorDocker2)); + } + + private Set<String> generateIPs(int start, int count) { + // Allow 4 containers + Set<String> additionalIps = new HashSet<>(); + for (int i = start; i < (start + count); i++) { + additionalIps.add("::" + i); + } + return additionalIps; + } + + private ApplicationId app(String tenant) { + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName("test") + .instanceName("default").build(); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 89119441a93..24e9736906d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -14,7 +14,6 @@ import org.junit.Test; import java.util.HashSet; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java new file mode 100644 index 00000000000..4dff3cff60c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -0,0 +1,139 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.path.Path; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author mortent + */ +public class DynamicDockerProvisioningTest { + + @Test + public void spare_capacity_used_only_when_replacement() { + // Use spare capacity only when replacement (i.e one node is failed) + // Test should allocate as much capacity as possible, verify that it is not possible to allocate one more unit + // Verify that there is still capacity (available spare) + // Fail one node and redeploy, Verify that one less node is empty. + + // Setup test + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); + enableDynamicAllocation(tester); + ApplicationId application1 = tester.makeApplicationId(); + tester.makeReadyNodes(5, "host-small", NodeType.host, 32); + deployZoneApp(tester); + Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); + + // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + DockerHostCapacity capacity = new DockerHostCapacity(tester.nodeRepository().getNodes(Node.State.values())); + assertThat(capacity.freeCapacityInFlavorEquivalence(flavor), greaterThan(0)); + + List<Node> initialSpareCapacity = findSpareCapacity(tester); + assertThat(initialSpareCapacity.size(), is(2)); + + try { + hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 4, 1, flavor.canonicalName()); + fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); + } catch (OutOfCapacityException e) { + } + + tester.fail(hosts.get(0)); + hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + List<Node> finalSpareCapacity = findSpareCapacity(tester); + + assertThat(finalSpareCapacity.size(), is(1)); + + // Uncomment the statement below to walk through the allocation events visually + //AllocationVisualizer.visualize(tester.getAllocationSnapshots()); + } + + @Test + public void non_prod_do_not_have_spares() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig()); + enableDynamicAllocation(tester); + tester.makeReadyNodes(3, "host-small", NodeType.host, 32); + deployZoneApp(tester); + Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3"); + + ApplicationId application1 = tester.makeApplicationId(); + List<HostSpec> hosts = tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")), + 3, 1, flavor.canonicalName()); + tester.activate(application1, ImmutableSet.copyOf(hosts)); + + List<Node> initialSpareCapacity = findSpareCapacity(tester); + assertThat(initialSpareCapacity.size(), is(0)); + } + + private List<Node> findSpareCapacity(ProvisioningTester tester) { + List<Node> nodes = tester.nodeRepository().getNodes(Node.State.values()); + NodeList nl = new NodeList(nodes); + return nodes.stream() + .filter(n -> n.type() == NodeType.host) + .filter(n -> nl.childNodes(n).size() == 0) // Nodes without children + .collect(Collectors.toList()); + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); + b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); + b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + return b.build(); + } + + private List<HostSpec> deployZoneApp(ProvisioningTester tester) { + ApplicationId applicationId = tester.makeApplicationId(); + List<HostSpec> list = tester.prepare(applicationId, + ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from("node-admin"), + Version.fromString("6.42")), + Capacity.fromRequiredNodeType(NodeType.host), + 1); + tester.activate(applicationId, ImmutableSet.copyOf(list)); + return list; + } + + private void enableDynamicAllocation(ProvisioningTester tester) { + tester.getCurator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClustersTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClustersTest.java new file mode 100644 index 00000000000..c4bd452d25c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorClustersTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +@SuppressWarnings("unchecked") +public class FlavorClustersTest { + + @Test + public void testSingletonClusters() { + NodeFlavors nodeFlavors = makeFlavors(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + FlavorClusters clusters = new FlavorClusters(nodeFlavors.getFlavors()); + Set<Set<Flavor>> expectedClusters = createExpectedClusters(nodeFlavors, + Collections.singletonList(0), Collections.singletonList(1), Collections.singletonList(2)); + assertEquals(expectedClusters, clusters.flavorClusters); + } + + @Test + public void testSingleClusterWithMultipleNodes() { + // 0 -> 1 -> 2 + NodeFlavors nodeFlavors = makeFlavors(Collections.singletonList(1), Collections.singletonList(2), Collections.emptyList()); + FlavorClusters clusters = new FlavorClusters(nodeFlavors.getFlavors()); + Set<Set<Flavor>> expectedClusters = createExpectedClusters(nodeFlavors, Arrays.asList(0, 1, 2)); + assertEquals(expectedClusters, clusters.flavorClusters); + } + + @Test + public void testMultipleClustersWithMultipleNodes() { + /* Creates flavors where 'replaces' graph that looks like this: + * 5 + * | + * | + * 3 4 8 + * \ / | + * \ / | + * 1 6 7 + * / \ + * / \ + * 0 2 + */ + NodeFlavors nodeFlavors = makeFlavors( + Collections.singletonList(1), // 0 -> {1} + Arrays.asList(3, 4), // 1 -> {3, 4} + Collections.singletonList(1), // 2 -> {1} + Collections.singletonList(5), // 3 -> {5} + Collections.emptyList(), // 4 -> {} + Collections.emptyList(), // 5 -> {} + Collections.emptyList(), // 6 -> {} + Collections.singletonList(8), // 7 -> {8} + Collections.emptyList()); // 8 -> {} + + FlavorClusters clusters = new FlavorClusters(nodeFlavors.getFlavors()); + Set<Set<Flavor>> expectedClusters = createExpectedClusters(nodeFlavors, + Arrays.asList(0, 1, 2, 3, 4, 5), + Collections.singletonList(6), + Arrays.asList(7, 8)); + assertEquals(expectedClusters, clusters.flavorClusters); + } + + private Set<Set<Flavor>> createExpectedClusters(NodeFlavors nodeFlavors, List<Integer>... clusters) { + return Arrays.stream(clusters).map(cluster -> + cluster.stream() + .map(flavorId -> nodeFlavors.getFlavorOrThrow("flavor-" + flavorId)) + .collect(Collectors.toSet())) + .collect(Collectors.toSet()); + } + + public static NodeFlavors makeFlavors(int numFlavors) { + FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); + for (int i = 0; i < numFlavors; i++) { + flavorConfigBuilder.addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); + } + return new NodeFlavors(flavorConfigBuilder.build()); + } + + /** + * Takes in variable number of List of Integers: + * For each list a flavor is created + * For each element, n, in list, the new flavor replace n'th flavor + */ + @SafeVarargs + public static NodeFlavors makeFlavors(List<Integer>... replaces) { + FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); + for (int i = 0; i < replaces.length; i++) { + FlavorsConfig.Flavor.Builder builder = flavorConfigBuilder + .addFlavor("flavor-" + i, 1. /* cpu*/, 3. /* mem GB*/, 2. /*disk GB*/, Flavor.Type.BARE_METAL); + + for (Integer replacesId : replaces[i]) { + flavorConfigBuilder.addReplaces("flavor-" + replacesId, builder); + } + } + return new NodeFlavors(flavorConfigBuilder.build()); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 3ecfae3312a..d849c1c3f77 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -337,6 +337,17 @@ public class ProvisioningTest { } @Test + public void deploy_specific_vespa_version() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.dev, RegionName.from("us-east"))); + + ApplicationId application = tester.makeApplicationId(); + tester.makeReadyNodes(4, "default"); + SystemState state = prepare(application, 2, 2, 3, 3, "default", Version.fromString("6.91"), tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + + @Test public void test_deployment_size() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.test, RegionName.from("us-east"))); @@ -681,11 +692,15 @@ public class ProvisioningTest { } private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, String flavor, ProvisioningTester tester) { + return prepare(application, container0Size, container1Size, content0Size, content1Size, flavor, Version.fromString("6.42"), tester); + } + + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, String flavor, Version wantedVersion, ProvisioningTester tester) { // "deploy prepare" with a two container clusters and a storage cluster having of two groups - ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), Version.fromString("6.42")); - ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("6.42")); - ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), Version.fromString("6.42")); - ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), Version.fromString("6.42")); + ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion); + ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion); + ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion); + ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion); Set<HostSpec> container0 = prepare(application, containerCluster0, container0Size, 1, flavor, tester); Set<HostSpec> container1 = prepare(application, containerCluster1, container1Size, 1, flavor, tester); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index bbf8967c254..5af13833a7a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -5,16 +5,18 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; @@ -23,8 +25,6 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; @@ -34,6 +34,7 @@ import java.io.IOException; import java.time.temporal.TemporalAmount; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -42,6 +43,7 @@ import java.util.Set; import java.util.UUID; import java.util.logging.Level; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -60,6 +62,7 @@ public class ProvisioningTester implements AutoCloseable { private final NodeRepositoryProvisioner provisioner; private final CapacityPolicies capacityPolicies; private final ProvisionLogger provisionLogger; + private final List<AllocationSnapshot> allocationSnapshots = new ArrayList<>(); public ProvisioningTester(Zone zone) { this(zone, createConfig()); @@ -75,7 +78,8 @@ public class ProvisioningTester implements AutoCloseable { this.clock = new ManualClock(); this.curator = curator; this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver); - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock, + (x,y) -> allocationSnapshots.add(new AllocationSnapshot(new NodeList(x), "Provision tester", y))); this.capacityPolicies = new CapacityPolicies(zone, nodeFlavors); this.provisionLogger = new NullProvisionLogger(); } @@ -111,6 +115,10 @@ public class ProvisioningTester implements AutoCloseable { //testingServer.close(); } + public List<AllocationSnapshot> getAllocationSnapshots() { + return allocationSnapshots; + } + public void advanceTime(TemporalAmount duration) { clock.advance(duration); } public NodeRepository nodeRepository() { return nodeRepository; } public ManualClock clock() { return clock; } @@ -213,13 +221,24 @@ public class ProvisioningTester implements AutoCloseable { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { + return makeReadyNodes(n, flavor, type, 0); + } + + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) { List<Node> nodes = new ArrayList<>(n); - for (int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { + Set<String> ips = IntStream.range(additionalIps * i, additionalIps * (i+1)) + .mapToObj(j -> String.format("127.0.0.%d", j)) + .collect(Collectors.toSet()); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - Optional.empty(), - nodeFlavors.getFlavorOrThrow(flavor), - type)); + UUID.randomUUID().toString(), + Collections.emptySet(), + ips, + Optional.empty(), + nodeFlavors.getFlavorOrThrow(flavor), + type)); + } nodes = nodeRepository.addNodes(nodes); nodes = nodeRepository.setDirty(nodes); return nodeRepository.setReady(nodes); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacityTest.java new file mode 100644 index 00000000000..7d32b1fff6d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacityTest.java @@ -0,0 +1,93 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; +import org.junit.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author smorgrav + */ +public class ResourceCapacityTest { + + @Test + public void basic_capacity_and_compare_operations() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); + b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); + b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER); + + FlavorsConfig flavors = b.build(); + Flavor hostLargeFlavor = new Flavor(flavors.flavor(0)); + Flavor hostSmallFlavor = new Flavor(flavors.flavor(1)); + Flavor d1Flavor = new Flavor(flavors.flavor(2)); + Flavor d2Flavor = new Flavor(flavors.flavor(3)); + Flavor d3Flavor = new Flavor(flavors.flavor(4)); + Flavor d3DiskFlavor = new Flavor(flavors.flavor(5)); + Flavor d3MemFlavor = new Flavor(flavors.flavor(6)); + Flavor d3CPUFlavor = new Flavor(flavors.flavor(7)); + + Node nodeHostSmall = node(hostSmallFlavor); + ResourceCapacity capacityOfHostSmall = new ResourceCapacity(nodeHostSmall); + + // Assert initial capacities + assertTrue(capacityOfHostSmall.hasCapacityFor(hostSmallFlavor)); + assertTrue(capacityOfHostSmall.hasCapacityFor(d1Flavor)); + assertTrue(capacityOfHostSmall.hasCapacityFor(d2Flavor)); + assertTrue(capacityOfHostSmall.hasCapacityFor(d3Flavor)); + assertFalse(capacityOfHostSmall.hasCapacityFor(hostLargeFlavor)); + + // Also check that we are taking all three resources into accout + assertFalse(capacityOfHostSmall.hasCapacityFor(d3DiskFlavor)); + assertFalse(capacityOfHostSmall.hasCapacityFor(d3MemFlavor)); + assertFalse(capacityOfHostSmall.hasCapacityFor(d3CPUFlavor)); + + // Compare it to various flavors + assertEquals(1, capacityOfHostSmall.compare(nodeCapacity(d1Flavor))); + assertEquals(1, capacityOfHostSmall.compare(nodeCapacity(d2Flavor))); + assertEquals(0, capacityOfHostSmall.compare(nodeCapacity(d3Flavor))); + assertEquals(-1, capacityOfHostSmall.compare(nodeCapacity(d3DiskFlavor))); + assertEquals(-1, capacityOfHostSmall.compare(nodeCapacity(d3CPUFlavor))); + assertEquals(-1, capacityOfHostSmall.compare(nodeCapacity(d3MemFlavor))); + + // Change free capacity and assert on rest capacity + capacityOfHostSmall.subtract(node(d1Flavor)); + assertEquals(0, capacityOfHostSmall.compare(nodeCapacity(d2Flavor))); + + // Assert on rest capacity + assertTrue(capacityOfHostSmall.hasCapacityFor(d1Flavor)); + assertFalse(capacityOfHostSmall.hasCapacityFor(d3Flavor)); + + // At last compare the disk and cpu and mem variations + assertEquals(-1, nodeCapacity(d3Flavor).compare(nodeCapacity(d3DiskFlavor))); + assertEquals(1, nodeCapacity(d3DiskFlavor).compare(nodeCapacity(d3CPUFlavor))); + assertEquals(-1, nodeCapacity(d3CPUFlavor).compare(nodeCapacity(d3MemFlavor))); + assertEquals(1, nodeCapacity(d3MemFlavor).compare(nodeCapacity(d3DiskFlavor))); + } + + private ResourceCapacity nodeCapacity(Flavor flavor) { + return new ResourceCapacity(node(flavor)); + } + + private Node node(Flavor flavor) { + return new Node("fake",Collections.singleton("127.0.0.1"), + Collections.emptySet(), "hostA", Optional.empty(), flavor, Status.initial(), + Node.State.ready, Optional.empty(), History.empty(), NodeType.host); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 1cb27df3972..d62deac2855 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -45,32 +45,6 @@ public class VirtualNodeProvisioningTest { } @Test - public void optimize_sorts_by_more_vms() { - List<Node> readyList = new ArrayList<>(); - readyList.addAll(tester.makeReadyNodes(2, flavor)); - readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost1")); - readyList.addAll(tester.makeReadyNodes(2, flavor)); - readyList.addAll(tester.makeReadyVirtualNodes(3, flavor, "parentHost2")); - readyList.addAll(tester.makeReadyVirtualNodes(2, flavor, "parentHost3")); - readyList.addAll(tester.makeReadyVirtualNodes(1, flavor, "parentHost4")); - readyList.addAll(tester.makeReadyNodes(3, flavor)); - assertEquals(15, readyList.size()); - List<Node> optimized = GroupPreparer.stripeOverHosts(readyList); - assertEquals(15, optimized.size()); - assertEquals("parentHost2", optimized.get(0).parentHostname().get()); - assertEquals("parentHost4", optimized.get(3).parentHostname().get()); - assertEquals("parentHost2", optimized.get(4).parentHostname().get()); - assertEquals("parentHost2", optimized.get(7).parentHostname().get()); - assertEquals(false, optimized.get(8).parentHostname().isPresent()); - assertEquals(false, optimized.get(9).parentHostname().isPresent()); - assertEquals(false, optimized.get(10).parentHostname().isPresent()); - assertEquals(false, optimized.get(11).parentHostname().isPresent()); - assertEquals(false, optimized.get(12).parentHostname().isPresent()); - assertEquals(false, optimized.get(13).parentHostname().isPresent()); - assertEquals(false, optimized.get(14).parentHostname().isPresent()); - } - - @Test public void distinct_parent_host_for_each_node_in_a_cluster() { tester.makeReadyVirtualNodes(2, flavor, "parentHost1"); tester.makeReadyVirtualNodes(2, flavor, "parentHost2"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index bad123aff82..9e46f11b6ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -149,6 +149,11 @@ public class RestApiTest { new byte[0], Request.Method.PUT), "{\"message\":\"Moved host6.yahoo.com to dirty\"}"); + // ... and set it back to ready as if this was from the node-admin with the temporary state rest api + assertResponse(new Request("http://localhost:8080/nodes/v2/state/availablefornewallocations/host6.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host6.yahoo.com to ready\"}"); + // Put a host in failed and make sure it's children are also failed assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/parent1.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host10.yahoo.com, host5.yahoo.com, parent1.yahoo.com to failed\"}"); @@ -191,6 +196,9 @@ public class RestApiTest { Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentDockerImage\": \"ignored-image-name:4443/vespa/ci:6.43.0\"}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index a7a2fe6b677..fea4fb8d4d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -28,6 +28,9 @@ "name":"OperatorChangeApplicationMaintainer" }, { + "name":"RetiredEarlyExpirer" + }, + { "name":"MetricsReporter" }, { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index e812b057953..3fbe1d900c5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -33,6 +33,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 1c1adb10d2b..f6368c58196 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -22,7 +22,7 @@ "clustertype": "container", "clusterid": "id1", "group": "0", - "index": 1, + "index": 0, "retired": false }, "restartGeneration": 0, @@ -38,6 +38,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json index 58bbd6f0128..5ff31ef24ca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -18,6 +18,7 @@ "failCount":0, "hardwareFailure":false, "wantToRetire":false, + "wantToDeprovision" : false, "history":[], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 387464324dd..faaff6337db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -33,6 +33,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json index 65531808404..dd9213b9110 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -31,6 +31,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 72d4ac75e72..cc907a2280c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -22,7 +22,7 @@ "clustertype": "container", "clusterid": "id1", "group": "0", - "index": 0, + "index": 1, "retired": false }, "restartGeneration": 0, @@ -39,6 +39,7 @@ "hardwareFailure": true, "hardwareFailureType": "memory_mcelog", "wantToRetire" : true, + "wantToDeprovision" : true, "history": [ { "event": "readied", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 105473e8247..e416634bbe5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -22,7 +22,7 @@ "clustertype": "container", "clusterid": "id1", "group": "0", - "index": 0, + "index": 1, "retired": false }, "restartGeneration": 0, @@ -36,6 +36,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json index 1cbb6be6213..21f1e28dff0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json @@ -18,6 +18,7 @@ "failCount": 1, "hardwareFailure": false, "wantToRetire": false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index 495e79787fd..f8e18a08ae6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -20,6 +20,7 @@ "failCount": 1, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json index eb8355626f7..864677137b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"deallocated","at":123,"agent":"system"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index ba9802da05f..6910c132206 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -33,6 +33,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json index 726e5627839..161ae522f64 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index ea040be7152..4c00c984881 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[], "ipAddresses":["127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index 0c894a5802d..70c20065194 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure" : false, "wantToRetire" : false, + "wantToDeprovision" : false, "history":[], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json index d96afb70967..41b807116db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure": false, "wantToRetire" : false, + "wantToDeprovision" : false, "history": [ { "event": "readied", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index 844364d34c5..92e33ef1f33 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -17,6 +17,7 @@ "failCount": 0, "hardwareFailure": false, "wantToRetire" : false, + "wantToDeprovision" : false, "history": [], "ipAddresses":["::1", "127.0.0.1"], "additionalIpAddresses":[] diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/HostStateChangeDenialReason.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/HostStateChangeDenialReason.java index 37366e6efd3..8345455d68a 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/HostStateChangeDenialReason.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/HostStateChangeDenialReason.java @@ -13,20 +13,16 @@ import java.util.Objects; public class HostStateChangeDenialReason { public static final String FIELD_NAME_CONSTRAINT = "constraint"; - public static final String FIELD_NAME_SERVICE_TYPE = "service-type"; public static final String FIELD_NAME_MESSAGE = "message"; private final String constraintName; - private final String serviceType; private final String message; @JsonCreator public HostStateChangeDenialReason( @JsonProperty(FIELD_NAME_CONSTRAINT) final String constraintName, - @JsonProperty(FIELD_NAME_SERVICE_TYPE) final String serviceType, @JsonProperty(FIELD_NAME_MESSAGE) final String message) { this.constraintName = constraintName; - this.serviceType = serviceType; this.message = message; } @@ -35,11 +31,6 @@ public class HostStateChangeDenialReason { return constraintName; } - @JsonProperty(FIELD_NAME_SERVICE_TYPE) - public String serviceType() { - return serviceType; - } - @JsonProperty(FIELD_NAME_MESSAGE) public String message() { return message; @@ -53,7 +44,6 @@ public class HostStateChangeDenialReason { final HostStateChangeDenialReason other = (HostStateChangeDenialReason) o; if (!Objects.equals(this.constraintName, other.constraintName)) return false; - if (!Objects.equals(this.serviceType, other.serviceType)) return false; if (!Objects.equals(this.message, other.message)) return false; return true; @@ -61,6 +51,6 @@ public class HostStateChangeDenialReason { @Override public int hashCode() { - return Objects.hash(constraintName, serviceType, message); + return Objects.hash(constraintName, message); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index e3451ad9576..1a1ffe13b37 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -60,6 +60,12 @@ public interface Orchestrator { void suspend(HostName hostName) throws HostStateChangeDeniedException, HostNameNotFoundException; /** + * Acquire permission to remove a node permanently from the application, or otherwise throw + * {@link OrchestrationException}. + */ + void acquirePermissionToRemove(HostName hostName) throws OrchestrationException; + + /** * Suspend normal operations for a group of nodes in the same application. * * @param nodeGroup The group of nodes in an application. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 72b644be464..5cd5f0ab638 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ApplicationApiImpl; @@ -124,6 +124,21 @@ public class OrchestratorImpl implements Orchestrator { suspendGroup(nodeGroup); } + @Override + public void acquirePermissionToRemove(HostName hostName) throws OrchestrationException { + ApplicationInstance<ServiceMonitorStatus> appInstance = getApplicationInstance(hostName); + NodeGroup nodeGroup = new NodeGroup(appInstance, hostName); + + try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly(appInstance.reference())) { + ApplicationApi applicationApi = new ApplicationApiImpl( + nodeGroup, + statusRegistry, + clusterControllerClientFactory); + + policy.acquirePermissionToRemove(applicationApi); + } + } + // Public for testing purposes @Override public void suspendGroup(NodeGroup nodeGroup) throws HostStateChangeDeniedException, HostNameNotFoundException { @@ -291,7 +306,7 @@ public class OrchestratorImpl implements Orchestrator { // If the clustercontroller throws an error the nodes will be marked as allowed to be down // and be set back up on next resume invocation. - setClusterStateInController(application, ClusterControllerState.MAINTENANCE); + setClusterStateInController(application, ClusterControllerNodeState.MAINTENANCE); } statusRegistry.setApplicationInstanceStatus(status); @@ -299,7 +314,7 @@ public class OrchestratorImpl implements Orchestrator { } private void setClusterStateInController(ApplicationInstance<ServiceMonitorStatus> application, - ClusterControllerState state) + ClusterControllerNodeState state) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException { // Get all content clusters for this application Set<ClusterId> contentClusterIds = application.serviceClusters().stream() diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java index 9de7bfab627..b51ecfd1331 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java @@ -13,13 +13,13 @@ public interface ClusterControllerClient { * * @throws IOException if there was a problem communicating with the cluster controller */ - ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException; + ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerNodeState wantedState) throws IOException; /** * Requests that a cluster controller sets all nodes in the cluster to the requested state. * * @throws IOException if there was a problem communicating with the cluster controller */ - ClusterControllerStateResponse setApplicationState(ClusterControllerState wantedState) throws IOException; + ClusterControllerStateResponse setApplicationState(ClusterControllerNodeState wantedState) throws IOException; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 01ae6dadfc7..87fb22b882c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -29,7 +29,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ * @throws IOException if there was a problem communicating with the cluster controller */ @Override - public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerNodeState wantedState) throws IOException { ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.SAFE); @@ -52,7 +52,7 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ * @throws IOException if there was a problem communicating with the cluster controller */ @Override - public ClusterControllerStateResponse setApplicationState(final ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setApplicationState(final ClusterControllerNodeState wantedState) throws IOException { final ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); final ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.FORCE); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerNodeState.java index e5db0ec9795..5886a44af02 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerNodeState.java @@ -11,13 +11,14 @@ import com.fasterxml.jackson.annotation.JsonValue; * * @author hakonhall */ -public enum ClusterControllerState { +public enum ClusterControllerNodeState { MAINTENANCE("maintenance"), - UP("up"); + UP("up"), + DOWN("down"); private final String wireName; - ClusterControllerState(final String wireName) { + ClusterControllerNodeState(final String wireName) { this.wireName = wireName; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java index 2ea489337d8..c3b4498a52d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java @@ -50,7 +50,7 @@ public class ClusterControllerStateRequest { public static class State { @JsonProperty("state") - public final ClusterControllerState state; + public final ClusterControllerNodeState state; /** * The reason the client is making the request to set the node state. @@ -59,7 +59,7 @@ public class ClusterControllerStateRequest { @JsonProperty("reason") public final String reason; - public State(ClusterControllerState state, String reason) { + public State(ClusterControllerNodeState state, String reason) { this.state = state; this.reason = reason; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java index 0ac602b106e..62c2bb372e3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.List; @@ -12,11 +13,19 @@ import java.util.List; public interface ApplicationApi { String applicationInfo(); + /** + * The policy acts on some subset of nodes in the application. + */ + NodeGroup getNodeGroup(); + List<ClusterApi> getClusters(); + ApplicationInstanceStatus getApplicationStatus(); + void setHostState(HostName hostName, HostStatus status); List<HostName> getNodesInGroupWithStatus(HostStatus status); + List<StorageNode> getStorageNodesInGroupInClusterOrder(); List<StorageNode> getUpStorageNodesInGroupInClusterOrder(); List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index 5f84c80a051..889455044b1 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry; @@ -67,12 +68,23 @@ public class ApplicationApiImpl implements ApplicationApi { @Override public List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() { + return getStorageNodesInGroupInClusterOrder().stream() + .filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) + .sorted(Comparator.reverseOrder()) + .collect(Collectors.toList()); + } + + @Override + public NodeGroup getNodeGroup() { + return nodeGroup; + } + + @Override + public List<StorageNode> getStorageNodesInGroupInClusterOrder() { return clusterInOrder.stream() .map(ClusterApi::storageNodeInGroup) .filter(Optional::isPresent) .map(Optional::get) - .filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) - .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); } @@ -86,6 +98,11 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override + public ApplicationInstanceStatus getApplicationStatus() { + return hostStatusService.getApplicationInstanceStatus(); + } + + @Override public void setHostState(HostName hostName, HostStatus status) { hostStatusService.setHostState(hostName, status); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java index 1698770904f..ffd3cf5b39e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNode.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; public interface StorageNode extends Comparable<StorageNode> { HostName hostName(); - void setNodeState(ClusterControllerState wantedState) throws HostStateChangeDeniedException; + void setNodeState(ClusterControllerNodeState wantedState) throws HostStateChangeDeniedException; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index 1984c5cca40..879c7b8f0e8 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; @@ -44,7 +44,7 @@ public class StorageNodeImpl implements StorageNode { } @Override - public void setNodeState(ClusterControllerState wantedNodeState) + public void setNodeState(ClusterControllerNodeState wantedNodeState) throws HostStateChangeDeniedException { // The "cluster name" used by the Cluster Controller IS the cluster ID. String clusterId = this.clusterId.s(); @@ -72,7 +72,6 @@ public class StorageNodeImpl implements StorageNode { throw new HostStateChangeDeniedException( hostName(), HostedVespaPolicy.CLUSTER_CONTROLLER_AVAILABLE_CONSTRAINT, - VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, "Failed to communicate with cluster controllers " + clusterControllers + ": " + e, e); } @@ -81,8 +80,7 @@ public class StorageNodeImpl implements StorageNode { throw new HostStateChangeDeniedException( hostName(), HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT, - VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, - "Failed to set state to " + wantedNodeState + " in controller: " + response.reason); + "Failed to set state to " + wantedNodeState + " in cluster controller: " + response.reason); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java index d0d7cd83e60..98062640ab0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java @@ -7,8 +7,14 @@ public interface ClusterPolicy { /** * There's an implicit group of nodes known to clusterApi. This method answers whether * it would be fine, just looking at this cluster (and disregarding Cluster Controller/storage - * which is handled separately), whether it would be fine to allow all services on all the nodes - * in the group to go down. + * which is handled separately), to allow all services on all the nodes in the group to go down. */ void verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException; + + /** + * There's an implicit group of nodes known to clusterApi. This method answers whether + * it would be fine, just looking at this cluster, to allow all services on all the nodes + * in the group to go down PERMANENTLY (removed from application). + */ + void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java index 9bd584d82af..6062fef7189 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java @@ -3,37 +3,33 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.OrchestrationException; +import com.yahoo.vespa.orchestrator.model.NodeGroup; + +import java.util.Optional; /** * @author bakksjo */ public class HostStateChangeDeniedException extends OrchestrationException { - private final String constraintName; - private final ServiceType serviceType; + private final Optional<ServiceType> serviceType; - public HostStateChangeDeniedException(HostName hostName, String constraintName, - ServiceType serviceType, String message) { - this(hostName, constraintName, serviceType, message, null); + public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) { + this(hostName, constraintName, message, null); } - public HostStateChangeDeniedException(HostName hostName, String constraintName, - ServiceType serviceType, String message, Throwable cause) { - this(hostName.toString(), constraintName, serviceType, message, cause); + public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) { + this(hostName.s(), constraintName, Optional.empty(), message, e); } - public HostStateChangeDeniedException(NodeGroup nodeGroup, - String constraintName, - ServiceType serviceType, - String message) { - this(nodeGroup.toCommaSeparatedString(), constraintName, serviceType, message, null); + public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) { + this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null); } private HostStateChangeDeniedException(String nodes, String constraintName, - ServiceType serviceType, + Optional<ServiceType> serviceType, String message, Throwable cause) { super(createMessage(nodes, constraintName, serviceType, message), cause); @@ -43,18 +39,18 @@ public class HostStateChangeDeniedException extends OrchestrationException { private static String createMessage(String nodes, String constraintName, - ServiceType serviceType, + Optional<ServiceType> serviceType, String message) { return "Changing the state of " + nodes + " would violate " + constraintName - + " for service type " + serviceType + ": " + message; + + (serviceType.isPresent() ? " for service type " + serviceType.get() : "") + + ": " + message; } public String getConstraintName() { return constraintName; } - public ServiceType getServiceType() { + public Optional<ServiceType> getServiceType() { return serviceType; } - } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index eb2fb9dc1b9..a081236f4bc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.orchestrator.model.ClusterApi; import static com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT; public class HostedVespaClusterPolicy implements ClusterPolicy { + @Override public void verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { if (clusterApi.noServicesOutsideGroupIsDown()) { @@ -25,8 +26,8 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { throw new HostStateChangeDeniedException( clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, - clusterApi.serviceType(), - "Suspension percentage would increase from " + clusterApi.percentageOfServicesDown() + "Suspension percentage for service type " + clusterApi.serviceType() + + " would increase from " + clusterApi.percentageOfServicesDown() + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() @@ -34,7 +35,35 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); } - static ConcurrentSuspensionLimitForCluster getConcurrentSuspensionLimit(ClusterApi clusterApi) { + @Override + public void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) + throws HostStateChangeDeniedException { + // This policy is similar to verifyGroupGoingDownIsFine, except that services being down in the group + // is no excuse to allow suspension (like it is for verifyGroupGoingDownIsFine), since if we grant + // suspension in this case they will permanently be down/removed. + + if (clusterApi.noServicesOutsideGroupIsDown()) { + return; + } + + int percentageOfServicesAllowedToBeDown = getConcurrentSuspensionLimit(clusterApi).asPercentage(); + if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= percentageOfServicesAllowedToBeDown) { + return; + } + + throw new HostStateChangeDeniedException( + clusterApi.getNodeGroup(), + ENOUGH_SERVICES_UP_CONSTRAINT, + "Down percentage for service type " + clusterApi.serviceType() + + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%." + + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription() + + " and these hosts are allowed to be down: " + + clusterApi.nodesAllowedToBeDownNotInGroupDescription()); + } + + // Non-private for testing purposes + ConcurrentSuspensionLimitForCluster getConcurrentSuspensionLimit(ClusterApi clusterApi) { if (VespaModelUtil.ADMIN_CLUSTER_ID.equals(clusterApi.clusterId())) { if (VespaModelUtil.SLOBROK_SERVICE_TYPE.equals(clusterApi.serviceType())) { return ConcurrentSuspensionLimitForCluster.ONE_NODE; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index 515a46cb1da..ac1587ce7de 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -5,12 +5,13 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ApplicationApiImpl; import com.yahoo.vespa.orchestrator.model.ClusterApi; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.StorageNode; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; @@ -23,6 +24,7 @@ import java.util.logging.Logger; public class HostedVespaPolicy implements Policy { + public static final String APPLICATION_SUSPENDED_CONSTRAINT = "application-suspended"; public static final String ENOUGH_SERVICES_UP_CONSTRAINT = "enough-services-up"; public static final String SET_NODE_STATE_CONSTRAINT = "controller-set-node-state"; public static final String CLUSTER_CONTROLLER_AVAILABLE_CONSTRAINT = "controller-available"; @@ -48,7 +50,7 @@ public class HostedVespaPolicy implements Policy { // Ask Cluster Controller to set UP storage nodes in maintenance. // These storage nodes are guaranteed to be NO_REMARKS for (StorageNode storageNode : application.getUpStorageNodesInGroupInClusterOrder()) { - storageNode.setNodeState(ClusterControllerState.MAINTENANCE); + storageNode.setNodeState(ClusterControllerNodeState.MAINTENANCE); log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set to MAINTENANCE"); } @@ -63,7 +65,7 @@ public class HostedVespaPolicy implements Policy { public void releaseSuspensionGrant(ApplicationApi application) throws HostStateChangeDeniedException { // Always defer to Cluster Controller whether it's OK to resume storage node for (StorageNode storageNode : application.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder()) { - storageNode.setNodeState(ClusterControllerState.UP); + storageNode.setNodeState(ClusterControllerNodeState.UP); log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set to UP"); } @@ -73,6 +75,36 @@ public class HostedVespaPolicy implements Policy { } } + @Override + public void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException { + ApplicationInstanceStatus applicationStatus = applicationApi.getApplicationStatus(); + if (applicationStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { + throw new HostStateChangeDeniedException( + applicationApi.getNodeGroup(), + HostedVespaPolicy.APPLICATION_SUSPENDED_CONSTRAINT, + "Unable to test availability constraints as the application " + applicationApi.applicationInfo() + + " is allowed to be down"); + } + + // Apply per-cluster policy + for (ClusterApi cluster : applicationApi.getClusters()) { + clusterPolicy.verifyGroupGoingDownPermanentlyIsFine(cluster); + } + + // Ask Cluster Controller to set storage nodes to DOWN. + // These storage nodes are guaranteed to be NO_REMARKS + for (StorageNode storageNode : applicationApi.getStorageNodesInGroupInClusterOrder()) { + storageNode.setNodeState(ClusterControllerNodeState.DOWN); + log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set DOWN"); + } + + // Ensure all nodes in the group are marked as allowed to be down + for (HostName hostName : applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { + applicationApi.setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); + log.log(LogLevel.INFO, hostName + " is now allowed to be down (suspended)"); + } + } + // TODO: Remove later - currently used for backward compatibility testing @Override public void grantSuspensionRequest(ApplicationInstance<ServiceMonitorStatus> applicationInstance, diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java index fee2940efda..2db842274b3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java @@ -30,6 +30,13 @@ public interface Policy { void releaseSuspensionGrant(ApplicationApi application) throws HostStateChangeDeniedException; /** + * Give all hosts in a group permission to be removed from the application. + * + * @param applicationApi + */ + void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException; + + /** * Release an earlier grant for suspension. * * @throws HostStateChangeDeniedException if the release failed. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index b6dac92206c..c6e9e5415ce 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -57,7 +57,7 @@ public class HostResource implements HostApi { throw new NotFoundException(e); } catch (HostStateChangeDeniedException e) { log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage()); - throw webExceptionWithDenialReason(hostName, e); + throw webExceptionWithDenialReason("suspend", hostName, e); } return new UpdateHostResponse(hostName.s(), null); } @@ -72,14 +72,19 @@ public class HostResource implements HostApi { throw new NotFoundException(e); } catch (HostStateChangeDeniedException e) { log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage()); - throw webExceptionWithDenialReason(hostName, e); + throw webExceptionWithDenialReason("resume", hostName, e); } return new UpdateHostResponse(hostName.s(), null); } - private static WebApplicationException webExceptionWithDenialReason(HostName hostName, HostStateChangeDeniedException e) { - HostStateChangeDenialReason hostStateChangeDenialReason = new HostStateChangeDenialReason( - e.getConstraintName(), e.getServiceType().s(), e.getMessage()); + private static WebApplicationException webExceptionWithDenialReason( + String operationDescription, + HostName hostName, + HostStateChangeDeniedException e) { + HostStateChangeDenialReason hostStateChangeDenialReason = + new HostStateChangeDenialReason( + e.getConstraintName(), + operationDescription + " failed: " + e.getMessage()); UpdateHostResponse response = new UpdateHostResponse(hostName.s(), hostStateChangeDenialReason); return new WebApplicationException( hostStateChangeDenialReason.toString(), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 09aa37f9610..ccee70630af 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; @@ -228,11 +227,10 @@ public class OrchestratorImplTest { Throwable supensionFailure = new HostStateChangeDeniedException( DummyInstanceLookupService.TEST6_HOST_NAME, "some-constraint", - new ServiceType("foo"), "error message"); doThrow(supensionFailure).when(orchestrator).suspendGroup(DummyInstanceLookupService.TEST6_NODE_GROUP); - doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", new ServiceType("foo1-service"), "foo1-message")) + doThrow(new HostStateChangeDeniedException(DummyInstanceLookupService.TEST1_HOST_NAME, "foo1-constraint", "foo1-message")) .when(orchestrator).resume(DummyInstanceLookupService.TEST1_HOST_NAME); doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST6_HOST_NAME); doNothing().when(orchestrator).resume(DummyInstanceLookupService.TEST3_HOST_NAME); @@ -251,10 +249,10 @@ public class OrchestratorImplTest { assertEquals("Failed to suspend NodeGroup{application=tenant-id-3:application-instance-3:prod:utopia-1:default, " + "hostNames=[test6.prod.us-east-1.vespahosted.ne1.yahoo.com]} with parent host parentHostname: " + "Changing the state of test6.prod.us-east-1.vespahosted.ne1.yahoo.com would violate " + - "some-constraint for service type foo: error message; With suppressed throwable " + + "some-constraint: error message; With suppressed throwable " + "com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException: " + - "Changing the state of test1.prod.utpoia-1.vespahosted.ut1.yahoo.com would violate " + - "foo1-constraint for service type foo1-service: foo1-message", + "Changing the state of test1.prod.utpoia-1.vespahosted.ut1.yahoo.com " + + "would violate foo1-constraint: foo1-message", e.getMessage()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java index 93dcb5815c6..a36a468a345 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -22,14 +22,14 @@ import java.util.Set; * @author smorgrav */ public class ClusterControllerClientFactoryMock implements ClusterControllerClientFactory { - Map<String, ClusterControllerState> nodes = new HashMap<>(); + Map<String, ClusterControllerNodeState> nodes = new HashMap<>(); public boolean isInMaintenance(ApplicationInstance<ServiceMonitorStatus> appInstance, HostName hostName) { try { ClusterId clusterName = VespaModelUtil.getContentClusterName(appInstance, hostName); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(appInstance, hostName); String globalMapKey = clusterName.s() + storageNodeIndex; - return nodes.getOrDefault(globalMapKey, ClusterControllerState.UP) == ClusterControllerState.MAINTENANCE; + return nodes.getOrDefault(globalMapKey, ClusterControllerNodeState.UP) == ClusterControllerNodeState.MAINTENANCE; } catch (Exception e) { //Catch all - meant to catch cases where the node is not part of a storage cluster return false; @@ -43,7 +43,7 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host); String globalMapKey = clusterName.s() + storageNodeIndex; - nodes.put(globalMapKey, ClusterControllerState.UP); + nodes.put(globalMapKey, ClusterControllerNodeState.UP); } } } @@ -53,13 +53,13 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie return new ClusterControllerClient() { @Override - public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setNodeState(int storageNodeIndex, ClusterControllerNodeState wantedState) throws IOException { nodes.put(clusterName + storageNodeIndex, wantedState); return new ClusterControllerStateResponse(true, "Yes"); } @Override - public ClusterControllerStateResponse setApplicationState(ClusterControllerState wantedState) throws IOException { + public ClusterControllerStateResponse setApplicationState(ClusterControllerNodeState wantedState) throws IOException { Set<String> keyCopy = new HashSet<>(nodes.keySet()); for (String s : keyCopy) { if (s.startsWith(clusterName)) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java index 68df4dce393..2841677ec04 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTest.java @@ -22,7 +22,7 @@ public class ClusterControllerClientTest { strategyMock, CLUSTER_NAME); - final ClusterControllerState wantedState = ClusterControllerState.MAINTENANCE; + final ClusterControllerNodeState wantedState = ClusterControllerNodeState.MAINTENANCE; clusterControllerClient.setNodeState(STORAGE_NODE_INDEX, wantedState); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java index cde644e7e8b..1081858812b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/SingleInstanceClusterControllerClientFactoryTest.java @@ -64,7 +64,7 @@ public class SingleInstanceClusterControllerClientFactoryTest { final List<HostName> clusterControllers = Arrays.asList(HOST_NAME_1); clientFactory.createClient(clusterControllers, "clusterName") - .setNodeState(0, ClusterControllerState.MAINTENANCE); + .setNodeState(0, ClusterControllerNodeState.MAINTENANCE); verify(jaxRsClientFactory).createClient( ClusterControllerJaxRsApi.class, @@ -90,7 +90,7 @@ public class SingleInstanceClusterControllerClientFactoryTest { final List<HostName> clusterControllers = Arrays.asList(HOST_NAME_1, HOST_NAME_2, HOST_NAME_3); clientFactory.createClient(clusterControllers, "clusterName") - .setNodeState(0, ClusterControllerState.MAINTENANCE); + .setNodeState(0, ClusterControllerNodeState.MAINTENANCE); verify(jaxRsClientFactory).createClient( eq(ClusterControllerJaxRsApi.class), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java new file mode 100644 index 00000000000..e6e60900906 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -0,0 +1,113 @@ +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.orchestrator.model.ClusterApi; +import com.yahoo.vespa.orchestrator.model.NodeGroup; +import com.yahoo.vespa.orchestrator.model.VespaModelUtil; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class HostedVespaClusterPolicyTest { + private ClusterApi clusterApi = mock(ClusterApi.class); + private HostedVespaClusterPolicy policy = spy(new HostedVespaClusterPolicy()); + + @Test + public void testSlobrokSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); + when(clusterApi.serviceType()).thenReturn(VespaModelUtil.SLOBROK_SERVICE_TYPE); + assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testAdminSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); + when(clusterApi.serviceType()).thenReturn(new ServiceType("non-slobrok-service-type")); + assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testStorageSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); + when(clusterApi.isStorageCluster()).thenReturn(true); + assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void testDefaultSuspensionLimit() { + when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); + when(clusterApi.isStorageCluster()).thenReturn(false); + assertEquals(ConcurrentSuspensionLimitForCluster.TEN_PERCENT, + policy.getConcurrentSuspensionLimit(clusterApi)); + } + + @Test + public void verifyGroupGoingDownIsFine_noServicesOutsideGroupIsDownIsFine() { + verifyGroupGoingDownIsFine(true, false, 13, true); + } + + @Test + public void verifyGroupGoingDownIsFine_noServicesInGroupIsUp() { + verifyGroupGoingDownIsFine(false, true, 13, true); + } + + @Test + public void verifyGroupGoingDownIsFine_percentageIsFine() { + verifyGroupGoingDownIsFine(false, false, 9, true); + } + + @Test + public void verifyGroupGoingDownIsFine_fails() { + verifyGroupGoingDownIsFine(false, false, 13, false); + } + + private void verifyGroupGoingDownIsFine(boolean noServicesOutsideGroupIsDown, + boolean noServicesInGroupIsUp, + int percentageOfServicesDownIfGroupIsAllowedToBeDown, + boolean expectSuccess) { + when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown); + when(clusterApi.noServicesInGroupIsUp()).thenReturn(noServicesInGroupIsUp); + when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); + doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi); + + when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); + when(clusterApi.percentageOfServicesDown()).thenReturn(5); + when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); + when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group"); + when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down"); + + NodeGroup nodeGroup = mock(NodeGroup.class); + when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); + when(nodeGroup.toCommaSeparatedString()).thenReturn("node-group"); + + when(clusterApi.noServicesInGroupIsUp()).thenReturn(false); + try { + policy.verifyGroupGoingDownIsFine(clusterApi); + if (!expectSuccess) { + fail(); + } + } catch (HostStateChangeDeniedException e) { + if (!expectSuccess) { + assertEquals("Changing the state of node-group would violate enough-services-up: " + + "Suspension percentage for service type service-type would increase from " + + "5% to 13%, over the limit of 10%. These instances may be down: " + + "services-down-and-not-in-group and these hosts are allowed to be down: " + + "allowed-to-be-down", e.getMessage()); + assertEquals("enough-services-up", e.getConstraintName()); + } + } + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index 5cba6375717..fe2d9a60acb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -2,53 +2,25 @@ package com.yahoo.vespa.orchestrator.policy; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; -import com.yahoo.vespa.orchestrator.TestUtil; +import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerState; -import com.yahoo.vespa.orchestrator.controller.ClusterControllerStateResponse; +import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ClusterApi; import com.yahoo.vespa.orchestrator.model.StorageNode; -import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; -import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; -import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.DOWN; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.NOT_CHECKED; -import static com.yahoo.vespa.service.monitor.ServiceMonitorStatus.UP; -import static org.fest.assertions.Assertions.assertThat; -import static org.junit.Assert.fail; + import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -105,8 +77,8 @@ public class HostedVespaPolicyTest { order.verify(clusterPolicy).verifyGroupGoingDownIsFine(clusterApi3); order.verify(applicationApi).getUpStorageNodesInGroupInClusterOrder(); - order.verify(storageNode1).setNodeState(ClusterControllerState.MAINTENANCE); - order.verify(storageNode3).setNodeState(ClusterControllerState.MAINTENANCE); + order.verify(storageNode1).setNodeState(ClusterControllerNodeState.MAINTENANCE); + order.verify(storageNode3).setNodeState(ClusterControllerNodeState.MAINTENANCE); order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); order.verify(applicationApi).setHostState(hostName1, HostStatus.ALLOWED_TO_BE_DOWN); @@ -116,689 +88,53 @@ public class HostedVespaPolicyTest { order.verifyNoMoreInteractions(); } - private static final TenantId TENANT_ID = new TenantId("tenantId"); - private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId"); - private static final HostName HOST_NAME_1 = new HostName("host-1"); - private static final HostName HOST_NAME_2 = new HostName("host-2"); - private static final HostName HOST_NAME_3 = new HostName("host-3"); - private static final HostName HOST_NAME_4 = new HostName("host-4"); - private static final HostName HOST_NAME_5 = new HostName("host-5"); - private static final ServiceType SERVICE_TYPE_1 = new ServiceType("service-1"); - private static final ServiceType SERVICE_TYPE_2 = new ServiceType("service-2"); - - // TODO: Replace HostedVespaClusterPolicy with ClusterPolicy mock - private final HostedVespaPolicy policy = - new HostedVespaPolicy(new HostedVespaClusterPolicy(), clientFactory); - - private final MutableStatusRegistry mutablestatusRegistry = mock(MutableStatusRegistry.class); - { - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); - } - - @Test - public void test_policy_everyone_agrees_everything_is_up() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP)) - .build(); - - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); - } - - private void grantWithAdminCluster( - ServiceMonitorStatus statusParallelInstanceOtherHost, - ServiceMonitorStatus statusInstanceOtherHost, - ServiceType serviceType, - boolean expectGranted) throws HostStateChangeDeniedException { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, statusInstanceOtherHost) - .instance(HOST_NAME_3, UP)) - .addCluster(new ClusterBuilder(VespaModelUtil.ADMIN_CLUSTER_ID, serviceType) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, statusParallelInstanceOtherHost)) - .build(); - - if (expectGranted) { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_1, HostStatus.ALLOWED_TO_BE_DOWN); - } else { - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_1, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - } - - @Test - public void test_parallel_cluster_down_is_ok() throws Exception { - grantWithAdminCluster(DOWN, UP, new ServiceType("some-service-type"), true); - } - - @Test - public void test_slobrok_cluster_down_is_not_ok() throws Exception { - grantWithAdminCluster(DOWN, UP, VespaModelUtil.SLOBROK_SERVICE_TYPE, false); - } - - @Test - public void test_other_cluster_instance_down_is_not_ok() throws Exception { - grantWithAdminCluster(DOWN, DOWN, new ServiceType("some-service-type"), false); - } - - @Test - public void test_all_up_is_ok() throws Exception { - grantWithAdminCluster(UP, UP, new ServiceType("some-service-type"), true); - } - - @Test - public void test_policy_other_host_allowed_to_be_down() { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void test_policy_this_host_allowed_to_be_down() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_3))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - - verify(mutablestatusRegistry, times(0)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void from_five_to_ten_percent_suspension() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - - verify(mutablestatusRegistry, times(1)).setHostState(HOST_NAME_3, HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void from_ten_to_fifteen_percent_suspension() { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void from_five_to_fifteen_percent_suspension() { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest( - applicationInstance, - HOST_NAME_3, - mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_1); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - @Test - public void test_policy_no_services() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder().build(); - - HostName hostName = new HostName("test-hostname"); - policy.grantSuspensionRequest( - applicationInstance, - hostName, - mutablestatusRegistry - ); - - verify(mutablestatusRegistry, times(1)).setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); - } - - // The cluster name happens to be the cluster id of any of the content service clusters. - private static final String CONTENT_CLUSTER_NAME = "content-cluster-id"; - private static final HostName CLUSTER_CONTROLLER_HOST = new HostName("controller-0"); - private static final HostName STORAGE_NODE_HOST = new HostName("storage-2"); - private static final int STORAGE_NODE_INDEX = 2; - - private static final ServiceCluster<ServiceMonitorStatus> CLUSTER_CONTROLLER_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId("cluster-0"), - VespaModelUtil.CLUSTER_CONTROLLER_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - TestUtil.clusterControllerConfigId(CONTENT_CLUSTER_NAME, 0), - CLUSTER_CONTROLLER_HOST, - UP))); - - private static final ServiceCluster<ServiceMonitorStatus> DISTRIBUTOR_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId(CONTENT_CLUSTER_NAME), - VespaModelUtil.DISTRIBUTOR_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - new ConfigId("distributor-id-1"), - new HostName("distributor-1"), - UP))); - - private static final ServiceCluster<ServiceMonitorStatus> STORAGE_SERVICE_CLUSTER = new ServiceCluster<>( - new ClusterId(CONTENT_CLUSTER_NAME), - VespaModelUtil.STORAGENODE_SERVICE_TYPE, - makeServiceInstanceSet( - new ServiceInstance<>( - TestUtil.storageNodeConfigId(CONTENT_CLUSTER_NAME, STORAGE_NODE_INDEX), - STORAGE_NODE_HOST, - UP))); - - private static final ApplicationInstance<ServiceMonitorStatus> APPLICATION_INSTANCE = - new ApplicationInstance<>( - TENANT_ID, - APPLICATION_INSTANCE_ID, - makeServiceClusterSet( - CLUSTER_CONTROLLER_SERVICE_CLUSTER, - DISTRIBUTOR_SERVICE_CLUSTER, - STORAGE_SERVICE_CLUSTER)); - - // The grantSuspensionRequest() and releaseSuspensionGrant() functions happen to have similar signature, - // which allows us to reuse test code for testing both functions. The actual call to one of these two functions - // is encapsulated into the following functional interface. - interface PolicyFunction { - void grant( - final HostedVespaPolicy policy, - final ApplicationInstance<ServiceMonitorStatus> applicationInstance, - final HostName hostName, - final MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException; - } - - /** - * Since grantSuspensionRequest and releaseSuspensionGrant is quite similar, this test util contains the bulk - * of the test code used to test their common functionality. - * - * @param grantFunction Encapsulates the grant function to call - * @param currentHostStatus The current HostStatus of the host - * @param expectedNodeStateSentToClusterController The NodeState the test expects to be sent to the controller, - * or null if no CC request is expected to be sent. - * @param expectedHostStateSetOnHostStatusService The HostState the test expects to be set on the host service. - */ - private void testCommonGrantFunctionality( - PolicyFunction grantFunction, - ApplicationInstance<ServiceMonitorStatus> application, - HostStatus currentHostStatus, - Optional<ClusterControllerState> expectedNodeStateSentToClusterController, - HostStatus expectedHostStateSetOnHostStatusService) throws Exception { - // There is only one service running on the host, which is a storage node. - // Therefore, the corresponding cluster controller will have to be contacted - // to ask for permission. - - ClusterControllerStateResponse response = new ClusterControllerStateResponse(true, "ok"); - // MOTD: anyInt() MUST be used for an int field, otherwise a NullPointerException is thrown! - // In general, special anyX() must be used for primitive fields. - when(client.setNodeState(anyInt(), any())).thenReturn(response); - - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(currentHostStatus); - - // Execution phase. - grantFunction.grant(policy, application, STORAGE_NODE_HOST, mutablestatusRegistry); - - // Verification phase. - - if (expectedNodeStateSentToClusterController.isPresent()) { - List<HostName> hostNames = CLUSTER_CONTROLLER_SERVICE_CLUSTER.serviceInstances().stream() - .map(service -> service.hostName()) - .collect(Collectors.toList()); - verify(clientFactory, times(1)) - .createClient( - hostNames, - CONTENT_CLUSTER_NAME); - verify(client, times(1)) - .setNodeState( - STORAGE_NODE_INDEX, - expectedNodeStateSentToClusterController.get()); - } else { - verify(client, never()).setNodeState(anyInt(), any()); - } - - verify(mutablestatusRegistry, times(1)) - .setHostState( - STORAGE_NODE_HOST, - expectedHostStateSetOnHostStatusService); - } - - @Test - public void test_defer_to_controller() throws Exception { - HostStatus currentHostStatus = HostStatus.NO_REMARKS; - ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.MAINTENANCE; - HostStatus expectedHostStateSetOnHostStatusService = HostStatus.ALLOWED_TO_BE_DOWN; - testCommonGrantFunctionality( - HostedVespaPolicy::grantSuspensionRequest, - APPLICATION_INSTANCE, - currentHostStatus, - Optional.of(expectedNodeStateSentToClusterController), - expectedHostStateSetOnHostStatusService); - } - - @Test - public void test_release_suspension_grant_gives_no_remarks() throws Exception { - HostStatus currentHostStatus = HostStatus.ALLOWED_TO_BE_DOWN; - ClusterControllerState expectedNodeStateSentToClusterController = ClusterControllerState.UP; - HostStatus expectedHostStateSetOnHostStatusService = HostStatus.NO_REMARKS; - testCommonGrantFunctionality( - HostedVespaPolicy::releaseSuspensionGrant, - APPLICATION_INSTANCE, - currentHostStatus, - Optional.of(expectedNodeStateSentToClusterController), - expectedHostStateSetOnHostStatusService); - } - - @Test - public void okToSuspendHostWithNoConfiguredServices() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, DOWN) - .instance(HOST_NAME_2, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void okToSuspendHostWithAllItsServicesDownEvenIfOthersAreDownToo() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void okToSuspendStorageNodeWhenStorageIsDown() throws Exception { - ServiceMonitorStatus storageNodeStatus = DOWN; - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new StorageClusterBuilder() - // DOWN storage service => ok to suspend and no cluster controller call - .instance(STORAGE_NODE_HOST, DOWN, STORAGE_NODE_INDEX) - .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) - .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) - .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) - .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) - // This service has one down service on another host, which should - // not block us from suspension because STORAGE_NODE_HOST down too. - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(STORAGE_NODE_HOST, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - HostStatus currentHostStatus = HostStatus.NO_REMARKS; - Optional<ClusterControllerState> dontExpectAnyCallsToClusterController = Optional.empty(); - testCommonGrantFunctionality( - HostedVespaPolicy::grantSuspensionRequest, - applicationInstance, - currentHostStatus, - dontExpectAnyCallsToClusterController, - HostStatus.ALLOWED_TO_BE_DOWN); - } - - @Test - public void denySuspendOfStorageIfOthersAreDown() throws Exception { - // If the storage service is up, but other hosts' storage services are down, - // we should be denied permission to suspend. This behavior is common for - // storage service and non-storage service (they differ when it comes to - // the cluster controller). - ServiceMonitorStatus storageNodeStatus = UP; - - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new StorageClusterBuilder() - .instance(STORAGE_NODE_HOST, storageNodeStatus, STORAGE_NODE_INDEX) - .instance(HOST_NAME_2, DOWN, STORAGE_NODE_INDEX + 1) - .instance(HOST_NAME_3, DOWN, STORAGE_NODE_INDEX + 2)) - .addCluster(CLUSTER_CONTROLLER_SERVICE_CLUSTER) - .addCluster(DISTRIBUTOR_SERVICE_CLUSTER) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(STORAGE_NODE_HOST, DOWN) - .instance(HOST_NAME_4, DOWN) - .instance(HOST_NAME_5, UP)) - .build(); - - when(mutablestatusRegistry.getHostStatus(any())).thenReturn(HostStatus.NO_REMARKS); - - try { - policy.grantSuspensionRequest(applicationInstance, STORAGE_NODE_HOST, mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); - } - } - - // In this test we verify the storage service cluster suspend policy of allowing at most 1 - // storage service to be effectively down. The normal policy (and the one used previously for storage) - // is to allow 10%. Therefore, the test verifies we disallow suspending 2 hosts = 5% (some random number <10%). - // - // Since the Orchestrator doesn't allow suspending the host, the Orchestrator doesn't even bother calling the - // Cluster Controller. The CC also has a policy of max 1, so it's just an optimization and safety guard. - @Test - public void dontBotherCallingClusterControllerIfOtherStorageNodesAreDown() throws Exception { - StorageClusterBuilder clusterBuilder = new StorageClusterBuilder(); - for (int i = 0; i < 40; ++i) { - clusterBuilder.instance(new HostName("host-" + i), UP, i); - } - ApplicationInstance<ServiceMonitorStatus> applicationInstance = - new AppBuilder().addCluster(clusterBuilder).build(); - - HostName host_1 = new HostName("host-1"); - when(mutablestatusRegistry.getHostStatus(eq(host_1))).thenReturn(HostStatus.NO_REMARKS); - - HostName host_2 = new HostName("host-2"); - when(mutablestatusRegistry.getHostStatus(eq(host_2))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - - try { - policy.grantSuspensionRequest(applicationInstance, host_1, mutablestatusRegistry); - fail(); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()) - .isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(VespaModelUtil.STORAGENODE_SERVICE_TYPE); - } - - verify(mutablestatusRegistry, never()).setHostState(any(), any()); - } - - @Test - public void ownServiceInstanceDown() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void ownServiceInstanceDown_otherServiceIsAllNotChecked() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, NOT_CHECKED) - .instance(HOST_NAME_4, NOT_CHECKED) - .instance(HOST_NAME_5, NOT_CHECKED)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } - - @Test - public void ownServiceInstanceDown_otherServiceIsAllNotChecked_oneHostDown() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, NOT_CHECKED) - .instance(HOST_NAME_4, NOT_CHECKED) - .instance(HOST_NAME_5, NOT_CHECKED)) - .build(); - - when(mutablestatusRegistry.getHostStatus(eq(HOST_NAME_4))).thenReturn(HostStatus.ALLOWED_TO_BE_DOWN); - try { - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - fail("Should not be allowed to set " + HOST_NAME_3 + " down when " + HOST_NAME_4 + " is already down."); - } catch (HostStateChangeDeniedException e) { - // As expected. - assertThat(e.getConstraintName()).isEqualTo(HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT); - assertThat(e.getServiceType()).isEqualTo(SERVICE_TYPE_2); - } - } + public void testAcquirePermissionToRemove() throws OrchestrationException { + final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); + final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory); + final ApplicationApi applicationApi = mock(ApplicationApi.class); + when(applicationApi.applicationInfo()).thenReturn("tenant:app"); - @Test - public void ownServiceInstanceDown_otherServiceIsAllUp() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, DOWN) - .instance(HOST_NAME_3, DOWN)) - .addCluster(new ClusterBuilder(SERVICE_TYPE_2) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_4, UP) - .instance(HOST_NAME_5, UP)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } + ClusterApi clusterApi1 = mock(ClusterApi.class); + ClusterApi clusterApi2 = mock(ClusterApi.class); + ClusterApi clusterApi3 = mock(ClusterApi.class); + List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + when(applicationApi.getClusters()).thenReturn(clusterApis); - @Test - public void hostHasTwoInstances_oneDownOneUp() throws Exception { - final ApplicationInstance<ServiceMonitorStatus> applicationInstance = new AppBuilder() - .addCluster(new ClusterBuilder(SERVICE_TYPE_1) - .instance(HOST_NAME_1, UP) - .instance(HOST_NAME_2, UP) - .instance(HOST_NAME_3, UP) - .instance(HOST_NAME_3, DOWN)) - .build(); - - policy.grantSuspensionRequest(applicationInstance, HOST_NAME_3, mutablestatusRegistry); - } + StorageNode storageNode1 = mock(StorageNode.class); + HostName hostName1 = new HostName("storage-1"); + when(storageNode1.hostName()).thenReturn(hostName1); - // Helper classes for terseness. + HostName hostName2 = new HostName("host-2"); - private static class AppBuilder { - private final Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>(); + StorageNode storageNode3 = mock(StorageNode.class); + HostName hostName3 = new HostName("storage-3"); + when(storageNode1.hostName()).thenReturn(hostName3); - public AppBuilder addCluster(final ServiceCluster<ServiceMonitorStatus> cluster) { - serviceClusters.add(cluster); - return this; - } + List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3); + when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); - public AppBuilder addCluster(final ClusterBuilder clusterBuilder) { - serviceClusters.add(clusterBuilder.build()); - return this; - } + List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames); - public AppBuilder addCluster(final StorageClusterBuilder clusterBuilder) { - serviceClusters.add(clusterBuilder.build()); - return this; - } + InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); - public ApplicationInstance<ServiceMonitorStatus> build() { - return new ApplicationInstance<>( - TENANT_ID, - APPLICATION_INSTANCE_ID, - serviceClusters); - } - } + policy.acquirePermissionToRemove(applicationApi); - private static class ClusterBuilder { - private final ServiceType serviceType; - private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>(); - private final ClusterId clusterId; - private int instanceIndex = 0; - - public ClusterBuilder(final ClusterId clusterId, final ServiceType serviceType) { - this.clusterId = clusterId; - this.serviceType = serviceType; - } - - public ClusterBuilder(final ServiceType serviceType) { - this.clusterId = new ClusterId("clusterId"); - this.serviceType = serviceType; - } - - public ClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status) { - instances.add(new ServiceInstance<>(new ConfigId("configId-" + instanceIndex), hostName, status)); - ++instanceIndex; - return this; - } - - public ServiceCluster<ServiceMonitorStatus> build() { - return new ServiceCluster<>(clusterId, serviceType, instances); - } - } + order.verify(applicationApi).getClusters(); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi1); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi2); + order.verify(clusterPolicy).verifyGroupGoingDownPermanentlyIsFine(clusterApi3); - private static class StorageClusterBuilder { - private final Set<ServiceInstance<ServiceMonitorStatus>> instances = new HashSet<>(); + order.verify(applicationApi).getStorageNodesInGroupInClusterOrder(); + order.verify(storageNode1).setNodeState(ClusterControllerNodeState.DOWN); + order.verify(storageNode3).setNodeState(ClusterControllerNodeState.DOWN); - public StorageClusterBuilder instance(final HostName hostName, final ServiceMonitorStatus status, int index) { - instances.add(new ServiceInstance<>(TestUtil.storageNodeConfigId("content", index), hostName, status)); - return this; - } + order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); + order.verify(applicationApi).setHostState(hostName1, HostStatus.ALLOWED_TO_BE_DOWN); + order.verify(applicationApi).setHostState(hostName2, HostStatus.ALLOWED_TO_BE_DOWN); + order.verify(applicationApi).setHostState(hostName3, HostStatus.ALLOWED_TO_BE_DOWN); - public ServiceCluster<ServiceMonitorStatus> build() { - return new ServiceCluster<>(new ClusterId(CONTENT_CLUSTER_NAME), VespaModelUtil.STORAGENODE_SERVICE_TYPE, instances); - } + order.verifyNoMoreInteractions(); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index 793b73ec5d8..3d3117f9e07 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorImpl; @@ -104,6 +103,10 @@ public class HostResourceTest { } @Override + public void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException { + } + + @Override public void releaseSuspensionGrant( ApplicationInstance<ServiceMonitorStatus> applicationInstance, HostName hostName, @@ -203,6 +206,11 @@ public class HostResourceTest { } @Override + public void acquirePermissionToRemove(ApplicationApi applicationApi) throws HostStateChangeDeniedException { + doThrow(); + } + + @Override public void releaseSuspensionGrant( ApplicationInstance<ServiceMonitorStatus> applicationInstance, HostName hostName, @@ -214,7 +222,6 @@ public class HostResourceTest { throw new HostStateChangeDeniedException( new HostName("some-host"), "impossible-policy", - new ServiceType("silly-service"), "This policy rejects all requests"); } } diff --git a/persistence/OWNERS b/persistence/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/persistence/OWNERS +++ b/persistence/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/persistence/src/tests/proxy/providerproxy_test.cpp b/persistence/src/tests/proxy/providerproxy_test.cpp index 514d9516c65..28ce33a2b5b 100644 --- a/persistence/src/tests/proxy/providerproxy_test.cpp +++ b/persistence/src/tests/proxy/providerproxy_test.cpp @@ -8,6 +8,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/persistence/proxy/providerproxy.h> #include <vespa/persistence/proxy/providerstub.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> diff --git a/persistence/src/tests/proxy/providerstub_test.cpp b/persistence/src/tests/proxy/providerstub_test.cpp index db218ef81b8..c750a3c94af 100644 --- a/persistence/src/tests/proxy/providerstub_test.cpp +++ b/persistence/src/tests/proxy/providerstub_test.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/persistence/proxy/buildid.h> #include <vespa/persistence/proxy/providerstub.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> @@ -15,6 +16,9 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/target.h> using document::BucketId; using document::ByteBuffer; diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index d2a3ab3be65..05e21096fef 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -9,6 +9,7 @@ #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/persistence/src/vespa/persistence/proxy/providerproxy.cpp b/persistence/src/vespa/persistence/proxy/providerproxy.cpp index 1f6d0b19be6..29b09af9953 100644 --- a/persistence/src/vespa/persistence/proxy/providerproxy.cpp +++ b/persistence/src/vespa/persistence/proxy/providerproxy.cpp @@ -8,9 +8,8 @@ #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> -#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/noncopyable.hpp> #include <vespa/fnet/frt/frt.h> #include <vespa/log/log.h> LOG_SETUP(".providerproxy"); diff --git a/persistence/src/vespa/persistence/proxy/providerstub.cpp b/persistence/src/vespa/persistence/proxy/providerstub.cpp index 38d7ffab5f5..b0df6fd54bb 100644 --- a/persistence/src/vespa/persistence/proxy/providerstub.cpp +++ b/persistence/src/vespa/persistence/proxy/providerstub.cpp @@ -9,10 +9,13 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/persistence/spi/persistenceprovider.h> -#include <persistence/spi/types.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/document/fieldset/fieldsetrepo.h> +#include <vespa/fnet/frt/values.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/rpcrequest.h> + using document::BucketId; using document::ByteBuffer; @@ -25,8 +28,7 @@ using vespalib::makeClosure; using vespalib::makeTask; using vespalib::nbostream; -namespace storage { -namespace spi { +namespace storage::spi { namespace { LoadType defaultLoadType(0, "default"); @@ -870,16 +872,13 @@ void ProviderStub::RPC_removeEntry(FRT_RPCRequest *req) { } void ProviderStub::SetupRpcCalls() { - FRT_ReflectionBuilder rb(&_supervisor); + FRT_ReflectionBuilder rb(_supervisor.get()); addConnect(rb, FRT_METHOD(ProviderStub::RPC_connect), this); - addInitialize( - rb, FRT_METHOD(ProviderStub::RPC_initialize), this); - addGetPartitionStates( - rb, FRT_METHOD(ProviderStub::RPC_getPartitionStates), this); + addInitialize(rb, FRT_METHOD(ProviderStub::RPC_initialize), this); + addGetPartitionStates(rb, FRT_METHOD(ProviderStub::RPC_getPartitionStates), this); addListBuckets(rb, FRT_METHOD(ProviderStub::RPC_listBuckets), this); addSetClusterState(rb, FRT_METHOD(ProviderStub::RPC_setClusterState), this); - addSetActiveState( - rb, FRT_METHOD(ProviderStub::RPC_setActiveState), this); + addSetActiveState(rb, FRT_METHOD(ProviderStub::RPC_setActiveState), this); addGetBucketInfo(rb, FRT_METHOD(ProviderStub::RPC_getBucketInfo), this); addPut(rb, FRT_METHOD(ProviderStub::RPC_put), this); addRemoveById(rb, FRT_METHOD(ProviderStub::RPC_removeById), this); @@ -889,12 +888,10 @@ void ProviderStub::SetupRpcCalls() { addGet(rb, FRT_METHOD(ProviderStub::RPC_get), this); addCreateIterator(rb, FRT_METHOD(ProviderStub::RPC_createIterator), this); addIterate(rb, FRT_METHOD(ProviderStub::RPC_iterate), this); - addDestroyIterator( - rb, FRT_METHOD(ProviderStub::RPC_destroyIterator), this); + addDestroyIterator(rb, FRT_METHOD(ProviderStub::RPC_destroyIterator), this); addCreateBucket(rb, FRT_METHOD(ProviderStub::RPC_createBucket), this); addDeleteBucket(rb, FRT_METHOD(ProviderStub::RPC_deleteBucket), this); - addGetModifiedBuckets( - rb, FRT_METHOD(ProviderStub::RPC_getModifiedBuckets), this); + addGetModifiedBuckets(rb, FRT_METHOD(ProviderStub::RPC_getModifiedBuckets), this); addSplit(rb, FRT_METHOD(ProviderStub::RPC_split), this); addJoin(rb, FRT_METHOD(ProviderStub::RPC_join), this); addMove(rb, FRT_METHOD(ProviderStub::RPC_move), this); @@ -905,23 +902,27 @@ void ProviderStub::SetupRpcCalls() { ProviderStub::ProviderStub(int port, uint32_t threads, const document::DocumentTypeRepo &repo, PersistenceProviderFactory &factory) - : _supervisor(), + : _supervisor(std::make_unique<FRT_Supervisor>()), _executor(threads, 256*1024), _repo(&repo), _factory(factory), _provider(), - _providerCleanupTask(_supervisor.GetScheduler(), _executor, _provider) + _providerCleanupTask(_supervisor->GetScheduler(), _executor, _provider) { SetupRpcCalls(); - _supervisor.SetSessionFiniHook(FRT_METHOD(ProviderStub::HOOK_fini), this); - _supervisor.Start(); - _supervisor.Listen(port); + _supervisor->SetSessionFiniHook(FRT_METHOD(ProviderStub::HOOK_fini), this); + _supervisor->Start(); + _supervisor->Listen(port); } ProviderStub::~ProviderStub() { - _supervisor.ShutDown(true); + _supervisor->ShutDown(true); sync(); } -} // namespace spi -} // namespace storage +int +ProviderStub::getPort() const { + return _supervisor->GetListenPort(); +} + +} diff --git a/persistence/src/vespa/persistence/proxy/providerstub.h b/persistence/src/vespa/persistence/proxy/providerstub.h index ce831547ef9..eff3560cf20 100644 --- a/persistence/src/vespa/persistence/proxy/providerstub.h +++ b/persistence/src/vespa/persistence/proxy/providerstub.h @@ -2,11 +2,14 @@ #pragma once -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/fnet/frt/invokable.h> +#include <vespa/fnet/task.h> #include <memory> +class FRT_Supervisor; + namespace document { class DocumentTypeRepo; } namespace storage { @@ -36,7 +39,7 @@ private: } }; - FRT_Supervisor _supervisor; + std::unique_ptr<FRT_Supervisor> _supervisor; vespalib::ThreadStackExecutor _executor; const document::DocumentTypeRepo *_repo; PersistenceProviderFactory &_factory; @@ -81,8 +84,8 @@ public: PersistenceProviderFactory &factory); ~ProviderStub(); - int getPort() const { return _supervisor.GetListenPort(); } bool hasClient() const { return (_provider.get() != 0); } + int getPort() const; void setRepo(const document::DocumentTypeRepo &repo) { _repo = &repo; } diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h index ad869c0add3..eb7a5daeb5a 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.h +++ b/persistence/src/vespa/persistence/spi/bucketinfo.h @@ -12,8 +12,7 @@ namespace vespalib { class asciistream; } -namespace storage { -namespace spi { +namespace storage::spi { class BucketInfo { public: @@ -105,6 +104,4 @@ private: vespalib::asciistream& operator<<(vespalib::asciistream& out, const BucketInfo& info); std::ostream& operator<<(std::ostream& out, const BucketInfo& info); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index 70bc8724c96..841fef4620d 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -5,9 +5,9 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> -namespace storage { -namespace spi { +namespace storage::spi { ClusterState::ClusterState(const lib::ClusterState& state, uint16_t nodeIndex, @@ -99,5 +99,4 @@ void ClusterState::serialize(vespalib::nbostream& o) const { o << _distribution->serialize(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index c9ceda982e0..1061a06fd28 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -3,6 +3,7 @@ #include "docentry.h" #include <vespa/document/fieldvalue/document.h> #include <sstream> +#include <cassert> namespace storage { namespace spi { diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index 359f6af5164..93db642611f 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -5,6 +5,7 @@ #include "bucketinfo.h" #include "bucket.h" #include "docentry.h" +#include <vespa/document/bucket/bucketidlist.h> namespace storage::spi { @@ -196,7 +197,7 @@ private: class BucketIdListResult : public Result { public: - typedef document::BucketId::List List; + using List = document::bucket::BucketIdList; /** * Constructor used when there was an error listing the buckets. diff --git a/persistencetypes/OWNERS b/persistencetypes/OWNERS index 11a58a546b1..4cef58fa4f9 100644 --- a/persistencetypes/OWNERS +++ b/persistencetypes/OWNERS @@ -1,2 +1,2 @@ vekterli -dybis + diff --git a/persistencetypes/src/persistence/spi/types.h b/persistencetypes/src/persistence/spi/types.h index d88fc5eb5ae..95d2c443948 100644 --- a/persistencetypes/src/persistence/spi/types.h +++ b/persistencetypes/src/persistence/spi/types.h @@ -54,8 +54,7 @@ namespace document { return is; \ } \ -namespace storage { -namespace spi { +namespace storage::spi { /** * \class storage::spi::NodeIndex @@ -111,6 +110,4 @@ enum MaintenanceLevel { HIGH }; -} // spi -} // storage - +} @@ -875,6 +875,11 @@ <version>${jetty.version}</version> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <version>1.3</version> @@ -1061,7 +1066,7 @@ <curator.version>2.9.1</curator.version> <jackson2.version>2.8.3</jackson2.version> <jersey2.version>2.23.2</jersey2.version> - <jetty.version>9.4.5.v20170502</jetty.version> + <jetty.version>9.4.6.v20170531</jetty.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <test.hide>true</test.hide> @@ -1077,7 +1082,6 @@ <module>application-deploy-plugin</module> <module>application-model</module> <module>application-preprocessor</module> - <module>bundle-plugin</module> <module>bundle-plugin-test</module> <module>chain</module> <module>clustercontroller-apps</module> @@ -1088,9 +1092,7 @@ <module>component</module> <module>config-application-package</module> <module>configdefinitions</module> - <module>configgen</module> <module>config-bundle</module> - <module>config-class-plugin</module> <module>config-lib</module> <module>config-model-api</module> <module>config-model-fat</module> @@ -1166,6 +1168,7 @@ <module>vdslib</module> <module>vespaclient-core</module> <module>vespaclient-container-plugin</module> + <module>vespaclient-java</module> <module>vespa-application-maven-plugin</module> <module>vespa-documentgen-plugin</module> <module>vespa_feed_perf</module> diff --git a/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ExampleProcessorTest.java b/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ExampleProcessorTest.java index 4aa03c4ed12..125cc5cbecc 100644 --- a/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ExampleProcessorTest.java +++ b/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ExampleProcessorTest.java @@ -9,7 +9,7 @@ import com.yahoo.processing.execution.Execution; import org.junit.Test; import static org.junit.Assert.assertThat; -import static org.junit.matchers.JUnitMatchers.containsString; +import static org.hamcrest.CoreMatchers.containsString; public class ExampleProcessorTest { diff --git a/sample-apps/basic-search-tensor/README.md b/sample-apps/basic-search-tensor/README.md new file mode 100644 index 00000000000..d7c72e20472 --- /dev/null +++ b/sample-apps/basic-search-tensor/README.md @@ -0,0 +1,4 @@ +# Sample app: basic search with tensors + +Basic search with tensors sample app. + diff --git a/sample-apps/basic-search-tensor/music-data-1.json b/sample-apps/basic-search-tensor/music-data-1.json new file mode 100644 index 00000000000..247745a5e32 --- /dev/null +++ b/sample-apps/basic-search-tensor/music-data-1.json @@ -0,0 +1,17 @@ +{ + "fields": { + "album": "Bad", + "artist": "Michael Jackson", + "title": "Bad", + "year": 1987, + "duration": 247, + "tensor_attribute": { + "cells": [ + { "address" : { "x" : "0" }, "value": 1.0 }, + { "address" : { "x" : "1" }, "value": 2.0 }, + { "address" : { "x" : "2" }, "value": 3.0 }, + { "address" : { "x" : "3" }, "value": 5.0 } + ] + } + } +} diff --git a/sample-apps/basic-search-tensor/music-data-2.json b/sample-apps/basic-search-tensor/music-data-2.json new file mode 100644 index 00000000000..d5419902574 --- /dev/null +++ b/sample-apps/basic-search-tensor/music-data-2.json @@ -0,0 +1,17 @@ +{ + "fields": { + "album": "Recovery", + "artist": "Eminem", + "title": "So Bad", + "year": 2010, + "tensor_attribute": { + "cells": [ + { "address" : { "x" : "0" }, "value": 2.0 }, + { "address" : { "x" : "1" }, "value": 3.0 }, + { "address" : { "x" : "2" }, "value": 4.0 }, + { "address" : { "x" : "3" }, "value": 6.0 } + ] + } + } +} + diff --git a/sample-apps/basic-search-tensor/pom.xml b/sample-apps/basic-search-tensor/pom.xml new file mode 100644 index 00000000000..e0b821272fc --- /dev/null +++ b/sample-apps/basic-search-tensor/pom.xml @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.example</groupId> + <artifactId>basic-search-java</artifactId> + <version>1.0.1</version> + <packaging>container-plugin</packaging> <!-- Use Vespa packaging --> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <test.hide>true</test.hide> + <!--<vespa_version>6-SNAPSHOT</vespa_version>--> + <vespa_version>6.101.9</vespa_version> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> <!-- Is this needed? --> + <version>${vespa_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> <!-- not container-dev --> + <version>${vespa_version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.1</version> + <configuration> + <optimize>true</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + <source>1.8</source> + <target>1.8</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.19.1</version> + <configuration> + <systemPropertyVariables> + <isMavenSurefirePlugin>true</isMavenSurefirePlugin> + </systemPropertyVariables> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> <!-- Zip the application package --> + <version>${vespa_version}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespa_version}</version> + <extensions>true</extensions> + </plugin> + </plugins> + </build> +</project> diff --git a/sample-apps/basic-search-tensor/src/main/application/constants/constant_tensor_file.json b/sample-apps/basic-search-tensor/src/main/application/constants/constant_tensor_file.json new file mode 100644 index 00000000000..9b20676627d --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/application/constants/constant_tensor_file.json @@ -0,0 +1,7 @@ +{ + "cells": [ + {"value": 0.0, "address": {"x": "0"}}, + {"value": 0.1, "address": {"x": "1"}}, + {"value": 0.2, "address": {"x": "2"}} + ] +} diff --git a/sample-apps/basic-search-tensor/src/main/application/hosts.xml b/sample-apps/basic-search-tensor/src/main/application/hosts.xml new file mode 100644 index 00000000000..3ab86a21aef --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/application/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/sample-apps/basic-search-tensor/src/main/application/search/query-profiles/types/root.xml b/sample-apps/basic-search-tensor/src/main/application/search/query-profiles/types/root.xml new file mode 100644 index 00000000000..e06f16c1032 --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/application/search/query-profiles/types/root.xml @@ -0,0 +1,3 @@ +<query-profile-type id="root" inherits="native"> + <field name="ranking.features.query(tensor)" type="tensor(x{})" /> +</query-profile-type> diff --git a/sample-apps/basic-search-tensor/src/main/application/searchdefinitions/music.sd b/sample-apps/basic-search-tensor/src/main/application/searchdefinitions/music.sd new file mode 100644 index 00000000000..85d59504321 --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/application/searchdefinitions/music.sd @@ -0,0 +1,55 @@ +search music { + document music { + field artist type string { + indexing: summary | index + } + + field artistId type string { + indexing: summary | attribute + } + + field title type string { + indexing: summary | index + } + + field album type string { + indexing: index + } + + field duration type int { + indexing: summary + } + + field year type int { + indexing: summary | attribute + } + + field popularity type int { + indexing: summary | attribute + } + + field tensor_attribute type tensor(x{}) { + indexing: summary | attribute + attribute: tensor(x{}) + } + + } + + constant tensor_constant { + file: constants/constant_tensor_file.json + type: tensor(x{}) + } + + rank-profile simple_tensor_ranking inherits default { + first-phase { + expression: sum(query(tensor) * attribute(tensor_attribute)) + } + } + + rank-profile constant_tensor_ranking inherits default { + first-phase { + expression: sum(query(tensor) * attribute(tensor_attribute) * constant(tensor_constant)) + } + } + +} diff --git a/sample-apps/basic-search-tensor/src/main/application/services.xml b/sample-apps/basic-search-tensor/src/main/application/services.xml new file mode 100644 index 00000000000..246ffff94de --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/application/services.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + + <jdisc version="1.0"> + <search> + <chain id="default" inherits="vespa"> + <searcher id="com.yahoo.example.ExampleTensorSearcher" bundle="basic-search-java" /> + </chain> + </search> + <document-api/> + <nodes> + <node hostalias="node1"/> + </nodes> + </jdisc> + + <content id="music" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index" /> + </documents> + <nodes> + <node hostalias="node1" distribution-key="0" /> + </nodes> + </content> + +</services> + diff --git a/sample-apps/basic-search-tensor/src/main/java/com/yahoo/example/ExampleTensorSearcher.java b/sample-apps/basic-search-tensor/src/main/java/com/yahoo/example/ExampleTensorSearcher.java new file mode 100644 index 00000000000..375ad7c98cb --- /dev/null +++ b/sample-apps/basic-search-tensor/src/main/java/com/yahoo/example/ExampleTensorSearcher.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.example; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.tensor.Tensor; + +public class ExampleTensorSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + + Object tensorProperty = query.properties().get("tensor"); + if (tensorProperty != null) { + + // Construct a Tensor object based on the query parameter + Tensor tensor = Tensor.from(tensorProperty.toString()); + + // Create a new rank feature using this tensor + query.getRanking().getFeatures().put("query(tensor)", tensor); + + // Set the rank profile to use + query.properties().set(new CompoundName("ranking"), "simple_tensor_ranking"); + } + + return execution.search(query); + } + +} diff --git a/sample-apps/blog-recommendation/README.md b/sample-apps/blog-recommendation/README.md index 0b6a00c9743..705c4e6879c 100644 --- a/sample-apps/blog-recommendation/README.md +++ b/sample-apps/blog-recommendation/README.md @@ -1,55 +1 @@ -Basic Search Application -================== -Start by [deploying a sample application](http://vespa.corp.yahoo.com/6/documentation/developing-with-vespa.html). - -### Find the endpoint - -When you have successfully deployed your own compiled version of the application above, you need to find the name of the "endpoint". -The endpoint is used for feeding and searching for data. -**Please allow a few minutes for the endpoint to appear after deployment** -You can find this endpoint by doing: - ```sh - - mvn vespa:endpoints | grep Endpoints - ``` - -You can also find it by looking at the [Hosted Vespa Dashboard](http://dashboard.vespa.corp.yahoo.com). - - -### Feed and search - 1. **Feed** the data that is to be searched - ```sh - - # Feeding two documents - curl -X POST --data-binary @music-data-1.json <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool - curl -X POST --data-binary @music-data-2.json <endpoint url>/document/v1/music/music/docid/2 | python -m json.tool - - ``` - -For feeding many documents fast and reliable, checkout [feeding example](https://git.corp.yahoo.com/vespa-samples/basic-feeding-client) - - 2. **Visit documents - - Since we do not have many documents we can list them all - ```sh - - # All documents - curl <endpoint url>/document/v1/music/music/docid | python -m json.tool - - # Document with id 1 - curl <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool - - ``` - - 3. **Search** - We can also search for documents: - ```sh - - curl '<endpoint url>/search/?query=bad' | python -m json.tool - - - ``` - -### Next step: from development to production -See [continuous deployments](http://vespa.corp.yahoo.com/6/documentation/continuous-deployment.html) for how to implement continuous deployments for production. -See [RESTified Document Operation API](http://vespa.corp.yahoo.com/6/documentation/document_api_v1.html) for documentation about the REST API for document operations. +Blog Recommendation diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig new file mode 100644 index 00000000000..9a536f38779 --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig @@ -0,0 +1,100 @@ +REGISTER vespa-hadoop.jar + +-- Create valid Vespa put operations +DEFINE VespaPutOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:blog_post::<post_id>', + 'create-tensor-fields=user_item_cf', + 'simple-array-fields=tags,categories' + ); + +DEFINE VespaPutOperationUser + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:user::<user_id>', + 'create-tensor-fields=user_item_cf', + 'simple-array-fields=has_read_items' + ); + +-- Transform tabular data to a Vespa document operation JSON format +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + +-- DEFINE VespaStorage +-- com.yahoo.vespa.hadoop.pig.VespaStorage( +-- 'create-document-operation=true', +-- 'operation=put', +-- 'docid=id:blog-recommendation:blog_post::<post_id>' +-- ); + +-- Load data from any source - here we load using PigStorage +data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}'); + +data_for_feed = FOREACH data GENERATE + date_gmt, + language, + author, + url, + title, + blog, + post_id, + tags, + blogname, + content, + categories; + +data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]'); + +data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id; +data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE + date_gmt AS date_gmt, + language AS language, + author AS author, + url AS url, + title AS title, + blog AS blog, + data_for_feed::post_id as post_id, + tags AS tags, + blogname AS blogname, + content AS content, + categories AS categories, + user_item_cf AS user_item_cf, + (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf; + +data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*); + +-- use cf latent factor +data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]'); +data_user = FOREACH data_user GENERATE + user_id AS user_id, + user_item_cf AS user_item_cf; + + +-- Articles already liked +data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); + +post_liked_per_user = GROUP data_likes BY uid; +post_liked_per_user = FOREACH post_liked_per_user GENERATE group AS user_id, data_likes.post_id AS has_read_items; + +-- Join user data +data_user = JOIN post_liked_per_user BY user_id FULL, data_user BY user_id; + +data_user = FOREACH data_user GENERATE (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id, + user_item_cf AS user_item_cf, + (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf, + has_read_items AS has_read_items; +data_user = FILTER data_user BY user_id IS NOT NULL; + +data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*); + +joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed; + +-- STORE data_for_feed into 'vespa_put_operation'; + +-- Store into Vespa +STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage(); + + + + diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig new file mode 100644 index 00000000000..59b173e16f4 --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig @@ -0,0 +1,71 @@ +REGISTER vespa-hadoop.jar + +-- UDF to create valid Vespa document operation in JSON format +DEFINE VespaPutOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-search:blog_post::<post_id>', + 'simple-array-fields=tags,categories' + ); + +-- UDF to send data to a Vespa endpoint +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + +-- Load data from any source - here we load using JsonLoader +data_first_release = LOAD 'blog-recommendation/first_release/trainPosts.json' USING + JsonLoader('date_gmt:chararray, + language:chararray, + author:chararray, + url:chararray, + title:chararray, + blog:chararray, + post_id:chararray, + tags:{T:(tag_name:chararray)}, + blogname:chararray, + date:chararray, + content:chararray, + categories:{T:(category_name:chararray)}, + likes:{T:(dt:chararray, uid:chararray)}'); + +data_second_release = LOAD 'blog-recommendation/second_release/trainPosts.json' USING + JsonLoader('date_gmt:chararray, + language:chararray, + author:chararray, + url:chararray, + title:chararray, + blog:chararray, + post_id:chararray, + tags:{T:(tag_name:chararray)}, + blogname:chararray, + date:chararray, + content:chararray, + categories:{T:(category_name:chararray)}, + likes:{T:(dt:chararray, uid:chararray)}'); + +data = UNION data_first_release, data_second_release; + +-- Select fields that will be sent to Vespa. +-- This should follow blog_post.sd +data_for_feed = FOREACH data GENERATE + date_gmt, + language, + author, + url, + title, + blog, + post_id, + tags, + blogname, + content, + categories; + +-- Create valid Vespa put operations in JSON format +data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*); + +-- Sample Vespa operations +-- data_for_feed_json_sample = SAMPLE data_for_feed_json 0.0005; +-- STORE data_for_feed_json_sample INTO 'blog-sample'; + +-- Store into Vespa +STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
\ No newline at end of file diff --git a/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig new file mode 100644 index 00000000000..fd06394c3af --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig @@ -0,0 +1,37 @@ +REGISTER vespa-hadoop.jar + +-- Create valid Vespa put operations +DEFINE VespaPutOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:blog_post::<post_id>', + 'create-tensor-fields=user_item_cf' + ); + +DEFINE VespaPutOperationUser + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:user::<user_id>', + 'create-tensor-fields=user_item_cf' + ); + +-- Transform tabular data to a Vespa document operation JSON format +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + + +data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]'); +data_doc_for_feed = FOREACH data_doc GENERATE VespaPutOperationDoc(*); + + +data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]'); +data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*); + + +-- Store into Vespa +STORE data_doc_for_feed INTO '$ENDPOINT' USING VespaStorage(); +STORE data_user_for_feed INTO '$ENDPOINT' USING VespaStorage(); + + + + diff --git a/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig new file mode 100644 index 00000000000..2e71dcbe9e3 --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig @@ -0,0 +1,15 @@ +-- Load data from any source - here we load using PigStorage +data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}'); + +data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); + +data_cf = FOREACH data_likes GENERATE uid, post_id, 1 as rate; + +data_cf = FILTER data_cf BY (uid IS NOT NULL) AND (uid != '') AND (post_id IS NOT NULL) AND (post_id != ''); + +-- data_cf_sample = SAMPLE data_cf 0.001; + +-- data_cf = LIMIT data_cf 10; + +STORE data_cf INTO 'blog-recommendation/trainPostsFinal_user_item_cf'; + diff --git a/sample-apps/blog-recommendation/src/pig/get_recommendations.pig b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig new file mode 100644 index 00000000000..00b03b0f49a --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig @@ -0,0 +1,29 @@ +-- REGISTER $VESPA_HADOOP_JAR +REGISTER vespa-hadoop.jar +-- REGISTER parquet-pig-bundle-1.8.1.jar + +-- Define Vespa query for retrieving blog posts +DEFINE BlogPostRecommendations + com.yahoo.vespa.hadoop.pig.VespaQuery( + 'query=http://ENDPOINT:8080/search/?user_id=<userid>&hits=100', + 'schema=rank:int,id:chararray,relevance:double,fields/post_id:chararray' + ); + +-- Load test_set data from a local file +test_set = LOAD 'data/cv/test_set_exploded' AS (post_id:chararray, userid:chararray); +users = FOREACH test_set GENERATE userid; +users = FILTER users BY userid IS NOT null; +users = DISTINCT users; + +users_limit = LIMIT users 10; + +-- Run a set of queries against Vespa +recommendations = FOREACH users_limit GENERATE userid, + FLATTEN(BlogPostRecommendations(*)) AS (rank, id, relevance, post_id); +recommendations = FOREACH recommendations GENERATE userid, rank, post_id; + +recommendations = FILTER recommendations BY rank IS NOT NULL AND post_id IS NOT NULL; + +-- Output recommendations +STORE recommendations INTO 'data/recommendations' USING PigStorage('\t', '-schema'); +-- STORE recommendations INTO 'data/recommendations' USING org.apache.parquet.pig.ParquetStorer(); diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig new file mode 100644 index 00000000000..4dac36a717f --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig @@ -0,0 +1,55 @@ +REGISTER '$VESPA_HADOOP_JAR' + +-- UDF to create valid Vespa document operation in JSON format +DEFINE VespaUpdateOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=update', + 'docid=id:blog-search:blog_post::<post_id>' + ); + +-- UDF to send data to a Vespa endpoint +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + +-- Load data from any source - here we load using JsonLoader +data = LOAD '$DATA_PATH' USING + JsonLoader('date_gmt:chararray, + language:chararray, + author:chararray, + url:chararray, + title:chararray, + blog:chararray, + post_id:chararray, + tags:{T:(tag_name:chararray)}, + blogname:chararray, + date:chararray, + content:chararray, + categories:{T:(category_name:chararray)}, + likes:{T:(dt:chararray, uid:chararray)}'); + +data = FILTER data BY likes IS NOT NULL; + +data_likes = FOREACH data GENERATE + blog, + post_id, + blogname, + FLATTEN(likes) AS (dt, uid); + +-- data_likes_limit = LIMIT data_likes 10; + +likes = FOREACH (GROUP data_likes ALL) + GENERATE COUNT(data_likes) as total_number; + +blog_popularity = FOREACH (GROUP data_likes BY blog) GENERATE + group as blog, + (double)COUNT(data_likes)/(double)likes.total_number AS popularity; + +data_update = JOIN data_likes BY blog, blog_popularity BY blog; +data_update = FOREACH data_update GENERATE + post_id, popularity; + +-- Create valid Vespa put operations in JSON format +data_for_feed_json = FOREACH data_update GENERATE VespaUpdateOperationDoc(*); + +-- Store into Vespa +STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage(); diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig new file mode 100644 index 00000000000..77943fd842a --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig @@ -0,0 +1,116 @@ +REGISTER '$VESPA_HADOOP_JAR' + +-- Create valid Vespa put operations +DEFINE VespaPutOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:blog_post::<post_id>', + 'create-tensor-fields=user_item_cf', + 'simple-array-fields=tags,categories' + ); + +DEFINE VespaPutOperationUser + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-recommendation:user::<user_id>', + 'create-tensor-fields=user_item_cf', + 'simple-array-fields=has_read_items' + ); + +-- Transform tabular data to a Vespa document operation JSON format +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + +-- Load data +data = LOAD '$DATA_PATH' USING + JsonLoader('date_gmt:chararray, + language:chararray, + author:chararray, + url:chararray, + title:chararray, + blog:chararray, + post_id:chararray, + tags:{T:(tag_name:chararray)}, + blogname:chararray, + date:chararray, + content:chararray, + categories:{T:(category_name:chararray)}, + likes:{T:(dt:chararray, uid:chararray)}'); + +data_for_feed = FOREACH data GENERATE + date_gmt, + language, + author, + url, + title, + blog, + post_id, + tags, + blogname, + content, + categories; + +-- Load Blog post CF latent factors +data_doc = LOAD '$BLOG_POST_FACTORS' USING + JsonLoader('post_id:chararray, + user_item_cf:[double]'); + +-- Join data and latent factors +data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id; +data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE + date_gmt AS date_gmt, + language AS language, + author AS author, + url AS url, + title AS title, + blog AS blog, + data_for_feed::post_id as post_id, + tags AS tags, + blogname AS blogname, + content AS content, + categories AS categories, + user_item_cf AS user_item_cf, + (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf; + +-- Generate valid Vespa JSON format +data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*); + +-- Load User CF latent factors +data_user = LOAD '$USER_FACTORS' USING + JsonLoader('user_id:chararray, + user_item_cf:[double]'); +data_user = FOREACH data_user GENERATE + user_id AS user_id, + user_item_cf AS user_item_cf; + +-- Articles already liked +data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); + +post_liked_per_user = GROUP data_likes BY uid; +post_liked_per_user = FOREACH post_liked_per_user GENERATE + group AS user_id, + data_likes.post_id AS has_read_items; + +-- Join user data +data_user = JOIN post_liked_per_user BY user_id FULL, + data_user BY user_id; + +data_user = FOREACH data_user GENERATE + (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id, + user_item_cf AS user_item_cf, + (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf, + has_read_items AS has_read_items; + +data_user = FILTER data_user BY user_id IS NOT NULL; + +-- Generate valid Vespa JSON format +data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*); + +joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed; + +-- Store into Vespa +STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage(); + + + + diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig new file mode 100644 index 00000000000..d20ccf505a9 --- /dev/null +++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig @@ -0,0 +1,51 @@ +REGISTER '$VESPA_HADOOP_JAR' +-- REGISTER vespa-hadoop.jar + +-- UDF to create valid Vespa document operation in JSON format +DEFINE VespaPutOperationDoc + com.yahoo.vespa.hadoop.pig.VespaDocumentOperation( + 'operation=put', + 'docid=id:blog-search:blog_post::<post_id>', + 'simple-array-fields=tags,categories' + ); + +-- UDF to send data to a Vespa endpoint +DEFINE VespaStorage + com.yahoo.vespa.hadoop.pig.VespaStorage(); + +-- Load data from any source - here we load using JsonLoader +data = LOAD '$DATA_PATH' USING + JsonLoader('date_gmt:chararray, + language:chararray, + author:chararray, + url:chararray, + title:chararray, + blog:chararray, + post_id:chararray, + tags:{T:(tag_name:chararray)}, + blogname:chararray, + date:chararray, + content:chararray, + categories:{T:(category_name:chararray)}, + likes:{T:(dt:chararray, uid:chararray)}'); + +-- Select fields that will be sent to Vespa. +-- This should follow blog_post.sd +data_for_feed = FOREACH data GENERATE + date_gmt, + language, + author, + url, + title, + blog, + post_id, + tags, + blogname, + content, + categories; + +-- Create valid Vespa put operations in JSON format +data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*); + +-- Store into Vespa +STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage(); diff --git a/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala new file mode 100644 index 00000000000..1a2c8f92730 --- /dev/null +++ b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala @@ -0,0 +1,59 @@ +import org.apache.spark.mllib.recommendation.ALS +import org.apache.spark.mllib.recommendation.MatrixFactorizationModel +import org.apache.spark.mllib.recommendation.Rating +import scala.util.parsing.json.JSONObject + +// Load and parse the data +val data = sc.textFile("blog-recommendation/trainPostsFinal_user_item_cf") +val ratings = data.map(_.split('\t') match { case Array(user, item, rate) => + Rating(user.toInt, item.toInt, rate.toDouble) +}) + +// Build the recommendation model using ALS +val rank = 10 +val numIterations = 10 +val model = ALS.train(ratings, rank, numIterations, 0.01) + +// Evaluate the model on rating data +val usersProducts = ratings.map { case Rating(user, product, rate) => + (user, product) +} +val predictions = + model.predict(usersProducts).map { case Rating(user, product, rate) => + ((user, product), rate) + } +val ratesAndPreds = ratings.map { case Rating(user, product, rate) => + ((user, product), rate) +}.join(predictions) +val MSE = ratesAndPreds.map { case ((user, product), (r1, r2)) => + val err = (r1 - r2) + err * err +}.mean() +println("Mean Squared Error = " + MSE) + +def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = { + + val id = modelFeatures._1 + val latentVector = modelFeatures._2 + var latentVectorMap:Map[String,Double] = Map() + var output:Map[String,Any] = Map() + + for ( i <- 0 until latentVector.length ){ + + latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i))) + + } + + output += ((id_string, id)) + output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap))) + + JSONObject(output) + +} + +val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id")) +product_features.saveAsTextFile("blog-recommendation/user_item_cf/product_features") +val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id")) +user_features.saveAsTextFile("blog-recommendation/user_item_cf/user_features") + + diff --git a/sample-apps/blog-recommendation/src/spark/data_exploration.scala b/sample-apps/blog-recommendation/src/spark/data_exploration.scala new file mode 100644 index 00000000000..228834cfb4b --- /dev/null +++ b/sample-apps/blog-recommendation/src/spark/data_exploration.scala @@ -0,0 +1,63 @@ +// sc is an existing SparkContext. +val sqlContext = new org.apache.spark.sql.SQLContext(sc) + +val original_train_post_path = "blog-recommendation-support/data/original_data/trainPosts.json" +val original_train_post_thin_path = "blog-recommendation-support/data/original_data/trainPostsThin.json" +val original_test_post_thin_path = "blog-recommendation-support/data/original_data/testPostsThin.json" + +val original_train_post = sqlContext.read.json(original_train_post_path) +val original_train_post_thin = sqlContext.read.json(original_train_post_thin_path) +val original_test_post_thin = sqlContext.read.json(original_test_post_thin_path) + +val count_original_train = original_train_post.count() +val count_original_train_thin = original_train_post_thin.count() +val count_original_test_thin = original_test_post_thin.count() + +// The inferred schema can be visualized using the printSchema() method. +original_train_post.printSchema() +original_train_post_thin.printSchema() +original_test_post_thin.printSchema() + +// No intersection between train and test data +original_train_post_thin.join(original_test_post_thin, original_train_post_thin("post_id") == original_test_post_thin("post_id")).count(2) + +// original_train_minimal_df +var original_train_minimal_df = original_train_post.select($"date_gmt", $"post_id", size($"likes").as("number_likes"), $"likes") +// no duplicate post_id +original_train_minimal_df.select("post_id").dropDuplicates().count() - original_train_minimal_df.select("post_id").count() + +// CHECK THIS DECISION - I SHOULD NOT EXLUDE POST_ID WITH ZERO LIKES +// OTHERWISE THERE WILL BE NO DOCUMENT IN THE TEST SET THAT NO ONE HAS LIKED, +// WHICH MAKES THE EXERCISE MUCH EASIER +// only post_id with at least one like +// original_train_minimal_df = original_train_minimal_df.filter("number_likes > 0") + +// Set some post_id aside to be present only on the test set +var sets = original_train_minimal_df.randomSplit(Array(0.95, 0.05), 123) + +var training_set = sets(0) +var test_set = sets(1) + +// flat dataframe so that each line is a combination of post_id and user +training_set = training_set.select($"post_id", explode($"likes").as("likes_flat")) +training_set = training_set.select("post_id", "likes_flat.uid") + +test_set = test_set.select($"post_id", explode($"likes").as("likes_flat")) +test_set = test_set.select("post_id", "likes_flat.uid") + +// randomly move some (post_id, uid) from training set to test set +sets = training_set.randomSplit(Array(0.85, 0.15), 123) + +training_set = sets(0) +var additional_test_set = sets(1) + +// concatenate test_set and additional_test_set +test_set = test_set.unionAll(additional_test_set) + +// see number of likes distribution +val like_dist = original_train_minimal_df.groupBy("number_likes").count().orderBy(asc("number_likes")).collect() +like_dist.map(println) + + + + diff --git a/sample-apps/blog-recommendation/src/spark/expected_percentile.scala b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala new file mode 100644 index 00000000000..986a3eb79f4 --- /dev/null +++ b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala @@ -0,0 +1,39 @@ +val test_file_path = "data/cv/test_set_exploded" +val blog_recom_file_path = "data/recommendations" +val size_recommendation_list = 100 + +val sqlContext = new org.apache.spark.sql.SQLContext(sc) + +val test_set = sc.textFile(test_file_path). + map(_.split("\t")).map(p => (p(0).toString, p(1).toString)). + toDF("post_id", "user_id") + +val recommendations = sc.textFile(blog_recom_file_path). + map(_.split("\t")).map(p => (p(0).toString, p(1).toString, p(2).toString)). + toDF("user_id", "rank", "post_id") + +// val recommendations = sqlContext.createDataFrame(Seq( +// ("16966742", "5", "1009088"), +// ("30463255", "10", "1044974") +// )).toDF("user_id", "rank", "post_id") + +// join data +var joined_data = test_set. + join(recommendations, + test_set("post_id") === recommendations("post_id") && + test_set("user_id") === recommendations("user_id")). + select(test_set("post_id"), + test_set("user_id"), + recommendations("rank")) + +// transform and add a column +joined_data = joined_data.withColumn("percentile", joined_data("rank")/size_recommendation_list) + +val expected_percentile = joined_data. + // groupBy($"user_id"). + groupBy(). + agg(sum($"percentile").as("sum_percentile"), + count($"post_id").as("number_read")). + withColumn("expected_percentile", $"sum_percentile" / $"number_read") + +expected_percentile.show()
\ No newline at end of file diff --git a/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala new file mode 100644 index 00000000000..0b76e8b8b1c --- /dev/null +++ b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala @@ -0,0 +1,60 @@ +import org.apache.spark.mllib.recommendation.ALS +import org.apache.spark.mllib.recommendation.MatrixFactorizationModel +import org.apache.spark.mllib.recommendation.Rating +import scala.util.parsing.json.JSONObject + +// Prepare data + +val data_path = "data/original_data/trainPosts.json" + +val sqlContext = new org.apache.spark.sql.SQLContext(sc) + +val full_dataset = sqlContext.read.json(data_path) + +var data = full_dataset.select($"post_id", explode($"likes").as("likes_flat")) +data = data.select($"likes_flat.uid".as("uid"), $"post_id") + +data = data.filter("uid is not null and uid != '' and post_id is not null and post_id != ''") + +val ratings = data.rdd.map(x => (x(0).toString, x(1).toString) match { + case (user, item) => Rating(user.toInt, item.toInt, 1) +}) + +// Train the model + +val rank = 10 +val numIterations = 10 +val model = ALS.train(ratings, rank, numIterations, 0.01) + +// Convert latent vectors from model to Vespa Tensor model + +def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = { + + val id = modelFeatures._1 + val latentVector = modelFeatures._2 + var latentVectorMap:Map[String,Double] = Map() + var output:Map[String,Any] = Map() + + for ( i <- 0 until latentVector.length ){ + + latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i))) + + } + + output += ((id_string, id)) + output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap))) + + JSONObject(output) + +} + +// Write user and item latent factors to disk + +val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id")) +product_features.saveAsTextFile("data/user_item_cf/product_features") +val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id")) +user_features.saveAsTextFile("data/user_item_cf/user_features") + + + + diff --git a/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala new file mode 100644 index 00000000000..2fc67734386 --- /dev/null +++ b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala @@ -0,0 +1,45 @@ +import org.apache.spark.sql.functions.udf + +// Inputs +val input_file_path = "data/original_data/trainPosts.json" +val test_perc_stage1 = 0.05 +val test_perc_stage2 = 0.15 +val training_file_path = "data/cv/training_set_exploded" +val test_file_path = "data/cv/test_set_exploded" +val seed = 123 + +val sqlContext = new org.apache.spark.sql.SQLContext(sc) + +// Load full dataset +val full_dataset = sqlContext.read.json(input_file_path) +val full_dataset_simple = full_dataset.select($"post_id", size($"likes").as("number_likes"), $"likes") + +// Set some blog posts aside to be present only on the test set +var sets = full_dataset_simple.randomSplit(Array(1 - test_perc_stage1, test_perc_stage1), seed) + +var training_set = sets(0) +val training_set_null = training_set.filter("number_likes == 0") +var training_set_exploded = training_set.select($"post_id", explode($"likes").as("likes_flat")) +training_set_exploded = training_set_exploded.select("post_id", "likes_flat.uid") + +var test_set = sets(1) +val test_set_null = test_set.filter("number_likes == 0") +var test_set_exploded = test_set.select($"post_id", explode($"likes").as("likes_flat")) +test_set_exploded = test_set_exploded.select("post_id", "likes_flat.uid") + +// randomly move some (post_id, uid) from training set to test set +sets = training_set_exploded.randomSplit(Array(1 - test_perc_stage2, test_perc_stage2), seed) + +training_set_exploded = sets(0) + +var additional_test_set_exploded = sets(1) +test_set_exploded = test_set_exploded.unionAll(additional_test_set_exploded) + +// concatenate exploded set with null set +val getNull = udf(() => None: Option[String]) +training_set_exploded = training_set_exploded.unionAll(training_set_null.select("post_id").withColumn("uid", getNull())) +test_set_exploded = test_set_exploded.unionAll(test_set_null.select("post_id").withColumn("uid", getNull())) + +// Write to disk +training_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(training_file_path) +test_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(test_file_path)
\ No newline at end of file diff --git a/sample-apps/blog-recommendation/training_data_example.json b/sample-apps/blog-recommendation/training_data_example.json new file mode 100644 index 00000000000..b25e4613fbe --- /dev/null +++ b/sample-apps/blog-recommendation/training_data_example.json @@ -0,0 +1,247 @@ +[ + { + "date_gmt": "2012-08-11 09:44:22", + "language": "en", + "author": "232", + "url": "https://joen.wordpress.com/?p=1339", + "title": "Selma and me", + "blog": "230", + "post_id": "522793", + "tags": [ + "selma" + ], + "blogname": "Lens Cap", + "date": "2012-08-11 09:44:22", + "content": "<a href=\"http://joen.files.wordpress.com/2012/08/wpid-imag03021.jpg\"><img title=\"\" class=\"alignnone\" alt=\"image\" src=\"http://joen.files.wordpress.com/2012/08/wpid-imag0302.jpg\" /></a>", + "categories": [ + "Photopool" + ], + "likes": [ + { + "dt": "2012-08-11 14:26:51", + "uid": "5" + }, + { + "dt": "2012-08-11 14:06:24", + "uid": "144" + }, + { + "dt": "2012-08-11 14:02:42", + "uid": "414033" + }, + { + "dt": "2012-08-11 10:56:57", + "uid": "6894686" + }, + { + "dt": "2012-08-11 14:16:09", + "uid": "20416304" + } + ] + }, + { + "date_gmt": "2012-07-17 16:05:35", + "language": "en", + "author": "322", + "url": "http://markjaquith.wordpress.com/?p=3894", + "title": "WP Help 1.0", + "blog": "316", + "post_id": "618834", + "tags": [], + "blogname": "Mark on WordPress", + "date": "2012-07-17 12:05:35", + "content": "My <a href=\"http://wordpress.org/extend/plugins/wp-help/\">WP Help</a> plugin just got a huge update. Version 1.0 is really worth checking out. I'm quite proud of it.\r\n\r\n<img src=\"http://markjaquith.files.wordpress.com/2012/07/screen-shot-2012-07-17-at-12-04-21-pm.png\" alt=\"\" title=\"Screen Shot 2012-07-17 at 12.04.21 PM\" width=\"584\" height=\"284\" class=\"aligncenter size-full wp-image-3896\" />\r\n\r\n<h3>WP Help</h3>\r\n\r\nWP Help is a plugin for creating documentation for display within the WordPress admin. Many WordPress installs are customized, and it's really helpful to have a centralized resource for documenting those features. You can create documents about creating content, editing content, moderating comments, or whatever you want! If you have clients who can't seem to remember how to do X, you should install WP Help and document it for them. WP Help is powered by WordPress Custom Post Types, so you create content using the full WordPress editor.\r\n\r\n<h3>Document Syncing</h3>\r\n\r\nOh yeah. This is the feature you've all been waiting for. If you have a standard set of help documents you want to use on multiple sites, this lets you do that. Create the documents, grab the (secret) sync URL for that site, and then plug that URL in to other sites. Those other sites will automatically pull down those documents, and keep them up-to-date (even handling new documents, deleted documents, renamed documents, and re-parented documents). Any internal links in the original document will be rewritten to be local to the destination WP Help install. So go ahead and use the WP internal linking functionality on your source site and know that those links will work on all the destination sites!\r\n\r\n<h3>Menu Placement</h3>\r\n\r\nThe menu item for the help documents can now be placed in one of four locations:\r\n\r\n<ol>\r\n<li>As a Dashboard submenu</li>\r\n<li>Top level, above the Dashboard</li>\r\n<li>Top level, below the Dashboard</li>\r\n<li>Top level, at the bottom</li>\r\n</ol>\r\n\r\nYou get a live preview of this (yeah, I know, super fancy).\r\n\r\n<h3>Menu Name</h3>\r\n\r\nThe menu name (and page title) can be changed. Just doubleclick the page title, edit it, and hit return. Boom. Again, this has a live preview.\r\n\r\n<h3>Topics List Name</h3>\r\n\r\nLikewise the topics list header can be renamed. Doubleclick, edit, return. Live preview.\r\n\r\n<h3>Edit Links & Management Links</h3>\r\n\r\nIf you can edit help documents, you'll get an edit link. When editing, there is a handy \"Manage\" link to jump you back to the documents management interface. Navigating in general has been improved quite a bit.\r\n\r\n<h3>Dashboard Widget</h3>\r\n\r\nThere is now a simple dashboard widget, listing all of your help documents.\r\n\r\n<h3>Better Default Access</h3>\r\n\r\nA lot of you said that you have Contributor-level users who need documentation just like authors do. So now they can view documentation by default (there's a hook if you want to change the capability required to view help documents).\r\n\r\n<h3>Lots of Little Tweaks</h3>\r\n\r\nThere are numerous little tweaks to improve your experience. <a href=\"http://wordpress.org/extend/plugins/wp-help/\">Check out out</a>!\r\n\r\n<h3>Roadmap</h3>\r\n\r\nThings I'm considering:\r\n\r\n<ul>\r\n<li>Restricting individual documents to users with a certain level of access</li>\r\n<li>Multiple sync sources</li>\r\n</ul>\r\n\r\nAny other ideas?", + "categories": [ + "WordPress" + ], + "likes": [ + { + "dt": "2012-07-17 21:12:13", + "uid": "1234" + }, + { + "dt": "2012-07-17 16:14:08", + "uid": "249119" + }, + { + "dt": "2012-07-17 18:52:43", + "uid": "5818061" + }, + { + "dt": "2012-07-20 18:00:27", + "uid": "10773334" + }, + { + "dt": "2012-07-18 10:27:04", + "uid": "23406451" + }, + { + "dt": "2012-07-17 18:50:47", + "uid": "37851431" + } + ] + }, + { + "date_gmt": "2012-07-13 20:49:59", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1214", + "title": "@ Café Meta", + "blog": "415", + "post_id": "1648262", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-07-13 16:49:59", + "content": "<p><a href=\"Café Meta\"><img src=\"https://is0.4sqi.net/pix/Nox6drHMgdWnt_7m8w1HaINMksVAtKXpLiD0Ocou-rg.jpg\"/></a></p>\n<p>Cheese filled pork rolls with mashed potatoes.</p>", + "categories": [ + "foursquare" + ], + "likes": [ + { + "dt": "2012-07-14 18:05:10", + "uid": "10916751" + } + ] + }, + { + "date_gmt": "2012-08-03 23:55:23", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1218", + "title": "@ El Campesino", + "blog": "415", + "post_id": "1194797", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-03 19:55:23", + "content": "<p><a href=\"http://4sq.com/7wXFrq\"><img src=\"https://is1.4sqi.net/pix/AwOlZW7UMMcfWhzioK7YCZIr-f2o1jJ-aTYTpxAuhHo.jpg\"/></a></p>\n<p>Celebrating a family birthday #burritofriday style.</p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-08-08 14:35:13", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1220", + "title": "@ Wild Goats Cafe", + "blog": "415", + "post_id": "285252", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-08 10:35:13", + "content": "<p><a href=\"http://4sq.com/aDGp5P\"><img src=\"https://is0.4sqi.net/pix/eylAdOrAsEbe3T0W7345n8nRfoiFv6GuYd4xQ4nu4g0.jpg\"/></a></p>\n<p>Italian Omelet\n with Grits</p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-08-09 00:00:54", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1222", + "title": "@ Oxford, OH", + "blog": "415", + "post_id": "250389", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-08 20:00:54", + "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is1.4sqi.net/pix/M75ofin4EhckhAszHPHLMzhC4mYWELShAnJKI12Re_c.jpg\"/></a></p>\n<p>Welcome to new home meal</p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-08-09 19:16:40", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1226", + "title": "@ IKEA", + "blog": "415", + "post_id": "1590141", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-09 15:16:40", + "content": "<p><a href=\"http://4sq.com/6IXrIh\"><img src=\"https://is0.4sqi.net/pix/FmrWa9WdnQi8q7LBd5iaI_oqRhBGyEHaxr4k_6qHETg.jpg\"/></a></p>\n<p></p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-08-11 17:00:36", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1228", + "title": "@ Oxford, OH", + "blog": "415", + "post_id": "583639", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-11 13:00:36", + "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is0.4sqi.net/pix/X6hSjNcr74mn5aWhEQ73M6Yzw2WFTcP9B4Y0bHrFrrw.jpg\"/></a></p>\n<p>Part of the property</p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-08-11 18:32:48", + "language": "en", + "author": "18342963", + "url": "http://justin.wordpress.com/?p=1230", + "title": "@ Phan-Shin", + "blog": "415", + "post_id": "1451283", + "tags": [ + "checkin" + ], + "blogname": "Justin G. Shreve", + "date": "2012-08-11 14:32:48", + "content": "<p><a href=\"http://4sq.com/90TF72\"><img src=\"https://is1.4sqi.net/pix/TMuuSbyftmgRT98vbeMgVCcMSBnsZzg-PNGRP4EqQqY.jpg\"/></a></p>\n<p>Last meal before leaving. General Tso's</p>", + "categories": [ + "foursquare" + ], + "likes": [] + }, + { + "date_gmt": "2012-07-12 17:03:49", + "language": "en", + "author": "486", + "url": "http://rick.wordpress.com/?p=1635", + "title": "On Baghdad by the Bay, July 12th at 8PM PDT; Interview with Dogcatcher and Wave Array", + "blog": "479", + "post_id": "1166238", + "tags": [ + "bay area music", + "music" + ], + "blogname": "STET", + "date": "2012-07-12 10:03:49", + "content": "<strong>Stream the show archive: [audio http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3] <a href=\"http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3\">Or download the MP3</a>.</strong>\r\n\r\n<a href=\"http://dogcatcherband.com/\"><img src=\"http://rick.files.wordpress.com/2012/07/dogcatcher4panel.jpg?w=450\" alt=\"\" title=\"Dogcatcher4panel\" width=\"450\" height=\"193\" class=\"alignnone size-large wp-image-1636\" /></a>\r\n<strong>Dogcatcher</strong> \r\n<a href=\"http://dogcatcherband.com/\">dogcatcherband.com</a>\r\n8pm\r\nThere’s something magical that happens when four musicians come together from four completely different backgrounds. For Mountain View’s Dogcatcher, the result is a mellow, subtly-intense blend of soul, jazz, folk and indie-rock. Their first album, KILR, was a quiet folk-oriented solo album Heine wrote while living in San Diego and digesting the Iraq War he’d recently returned from. After KILR, Heine returned to his hometown, Mountain View, and formed a proper group, plucking musicians out of totally different bands to create an eclectic, genre-defying sound. On Dogcatcher’s second album, It’s Easy, the four members demonstrate that they have truly come into their own. Heine’s soft, hushed voice and simple-stated lyrics compliment the easy, hip-shaking nature of the music. But for listeners looking for lyrics with depth, Heine will not disappoint. Rather than aim for overt expressions of emotion, Heine always goes for the understated, which makes the emotionality behind the music that much more powerful with repeat listens. \r\n\r\n[youtube http://www.youtube.com/watch?v=ZbI_KdDK8-c?rel=0&w=560&h=315]\r\n<strong>Wave Array </strong>\r\n<a href=\"http://wavearraymusic.com/\">wavearraymusic.com</a>\r\n9:20\r\nWave Array is a four-piece rock band hailing from the East Bay/San Francisco, CA that is stretching the limits of alternative/indie rock into the psychedelic and experimental realms. Using vibrant guitar and bass work, driving rhythms, and palpable melodies, the quartet aims to resonate frequencies of human emotion and thought buried beneath the preoccupation of everyday life. Wave Array's eclectic compositions of melodic vamps and hard hitting jams have already sparked the interest of the Bay Area music community as well as local and nation-wide radio. Their self-produced debut album, Cheapjack Moon, is available on iTunes and Amazon.", + "categories": [ + "Radio Valencia" + ], + "likes": [ + { + "dt": "2012-07-12 18:27:02", + "uid": "34944702" + } + ] + } +]
\ No newline at end of file diff --git a/sample-apps/boolean-search/README.md b/sample-apps/boolean-search/README.md index a13f54b7fb5..6f7f427b8f9 100644 --- a/sample-apps/boolean-search/README.md +++ b/sample-apps/boolean-search/README.md @@ -16,5 +16,5 @@ type predicate to the .sd-file. (Remember to set the arity parameter.) 2. **Search** using yql expressions, e.g. `select * from sources * where predicate(target, {"name":"Wile E. Coyote"},{});` ```sh -curl "<endpoint url>/search/?query=sddocname:ad&yql=select%20*%20from%20sources%20*%20where%20predicate(target%2C%20%7B%22name%22%3A%22Wile%20E.%20Coyote%22%7D%2C%7B%7D)%3B" + curl "<endpoint url>/search/?query=sddocname:ad&yql=select%20*%20from%20sources%20*%20where%20predicate(target%2C%20%7B%22name%22%3A%22Wile%20E.%20Coyote%22%7D%2C%7B%7D)%3B" ``` diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp index 50522aae3b9..f5b5b22d0d1 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcommon/attribute/config.h> diff --git a/searchcommon/src/tests/schema/schema_test.cpp b/searchcommon/src/tests/schema/schema_test.cpp index 9368e92f691..996ac7bcdde 100644 --- a/searchcommon/src/tests/schema/schema_test.cpp +++ b/searchcommon/src/tests/schema/schema_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vespalib/stllike/string.h> #include <fstream> diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp index dccfb6c6b27..8b58d2d50c2 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "search_context_params.h" +#include <cstdint> #include <limits> -namespace search { -namespace attribute { +namespace search::attribute { SearchContextParams::SearchContextParams() : _diversityAttribute(nullptr), @@ -16,4 +15,3 @@ SearchContextParams::SearchContextParams() } } -} diff --git a/searchcore/.gitignore b/searchcore/.gitignore index 6f79fc39b2d..555b6a2fddc 100644 --- a/searchcore/.gitignore +++ b/searchcore/.gitignore @@ -8,7 +8,6 @@ public_html .Install_completed .PreBuild_completed update.log - /target /pom.xml.build Makefile diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 4fc000e5a49..eaff8129234 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -103,6 +103,7 @@ vespa_define_module( src/tests/proton/feedtoken src/tests/proton/flushengine src/tests/proton/flushengine/prepare_restart_flush_strategy + src/tests/proton/flushengine/shrink_lid_space_flush_target src/tests/proton/index src/tests/proton/index/index_writer src/tests/proton/initializer @@ -133,7 +134,6 @@ vespa_define_module( src/tests/proton/reprocessing/document_reprocessing_handler src/tests/proton/reprocessing/reprocessing_runner src/tests/proton/server - src/tests/proton/server/data_directory_upgrader src/tests/proton/server/disk_mem_usage_filter src/tests/proton/server/health_adapter src/tests/proton/server/memory_flush_config_updater diff --git a/searchcore/src/apps/fdispatch/.gitignore b/searchcore/src/apps/fdispatch/.gitignore index 7ef8af69759..6cb8d53bdc1 100644 --- a/searchcore/src/apps/fdispatch/.gitignore +++ b/searchcore/src/apps/fdispatch/.gitignore @@ -1,2 +1 @@ -/fdispatch -fdispatch-bin +vespa-fdispatch-bin diff --git a/searchcore/src/apps/fdispatch/CMakeLists.txt b/searchcore/src/apps/fdispatch/CMakeLists.txt index 728f4e46883..63e1e6ca515 100644 --- a/searchcore/src/apps/fdispatch/CMakeLists.txt +++ b/searchcore/src/apps/fdispatch/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(searchcore_fdispatch_app SOURCES fdispatch.cpp - OUTPUT_NAME fdispatch-bin + OUTPUT_NAME vespa-fdispatch-bin INSTALL sbin DEPENDS searchcore_fdispatch_program diff --git a/searchcore/src/apps/fdispatch/fdispatch.cpp b/searchcore/src/apps/fdispatch/fdispatch.cpp index 799c42887da..9f6b13bc858 100644 --- a/searchcore/src/apps/fdispatch/fdispatch.cpp +++ b/searchcore/src/apps/fdispatch/fdispatch.cpp @@ -34,17 +34,17 @@ protected: return (vespalib::SignalHandler::INT.check() || vespalib::SignalHandler::TERM.check()); } - void Usage(void); + void Usage(); bool GetOptions(int *exitCode); public: int Main() override; - FastS_FDispatchApp(void); - ~FastS_FDispatchApp(void); + FastS_FDispatchApp(); + ~FastS_FDispatchApp(); }; -FastS_FDispatchApp::FastS_FDispatchApp(void) +FastS_FDispatchApp::FastS_FDispatchApp() { } @@ -174,7 +174,7 @@ FastS_FDispatchApp::GetOptions(int *exitCode) } void -FastS_FDispatchApp::Usage(void) +FastS_FDispatchApp::Usage() { printf("FAST Search - fdispatch %s\n", FastS_VersionTag); printf("\n" @@ -192,7 +192,7 @@ FastS_FDispatchApp::Usage(void) } -FastS_FDispatchApp::~FastS_FDispatchApp(void) +FastS_FDispatchApp::~FastS_FDispatchApp() { } diff --git a/searchcore/src/apps/proton/.gitignore b/searchcore/src/apps/proton/.gitignore index 4c244afd919..c2cc445d918 100644 --- a/searchcore/src/apps/proton/.gitignore +++ b/searchcore/src/apps/proton/.gitignore @@ -1,4 +1,3 @@ .depend Makefile -proton -proton-bin +vespa-proton-bin diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt index 13f04d2d6f4..c5398feed2e 100644 --- a/searchcore/src/apps/proton/CMakeLists.txt +++ b/searchcore/src/apps/proton/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchcore_proton_app SOURCES downpersistence.cpp proton.cpp - OUTPUT_NAME proton-bin + OUTPUT_NAME vespa-proton-bin INSTALL sbin DEPENDS searchcore_server diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 3509d59d635..b7872286246 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "downpersistence.h" + #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/searchlib/util/statefile.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> -#include "downpersistence.h" -#include <vespa/log/log.h> -LOG_SETUP(".downpersistence"); - -namespace storage { -namespace spi { +namespace storage::spi { namespace { @@ -18,18 +14,15 @@ Result errorResult(Result::FATAL_ERROR, "Node is down"); } - DownPersistence::DownPersistence(const vespalib::string &downReason) : _downReason(downReason) { } - DownPersistence::~DownPersistence() { } - Result DownPersistence::initialize() { @@ -63,7 +56,6 @@ DownPersistence:: setActiveState(const Bucket&, BucketInfo::ActiveState) return errorResult; } - BucketInfoResult DownPersistence:: getBucketInfo(const Bucket&) const { @@ -77,7 +69,6 @@ DownPersistence::put(const Bucket&, Timestamp, const Document::SP&, Context&) return errorResult; } - RemoveResult DownPersistence:: remove(const Bucket&, Timestamp, const DocumentId&, Context&) @@ -86,7 +77,6 @@ DownPersistence:: remove(const Bucket&, Timestamp, errorResult.getErrorMessage()); } - RemoveResult DownPersistence::removeIfFound(const Bucket&, Timestamp, const DocumentId&, Context&) @@ -95,14 +85,12 @@ DownPersistence::removeIfFound(const Bucket&, Timestamp, errorResult.getErrorMessage()); } - Result DownPersistence::removeEntry(const Bucket&, Timestamp, Context&) { return errorResult; } - UpdateResult DownPersistence::update(const Bucket&, Timestamp, const DocumentUpdate::SP&, Context&) { @@ -110,7 +98,6 @@ UpdateResult DownPersistence::update(const Bucket&, Timestamp, errorResult.getErrorMessage()); } - Result DownPersistence::flush(const Bucket&, Context&) { @@ -195,5 +182,3 @@ DownPersistence::move(const Bucket&, PartitionId, Context&) } } - -} diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index 4eeb301e617..27083a07d6b 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once -namespace storage { +#include <vespa/persistence/spi/persistenceprovider.h> -namespace spi { +namespace storage::spi { /* * Persistence provider that returns error code for all operations @@ -23,83 +24,34 @@ public: typedef std::unique_ptr<PersistenceProvider> UP; - virtual ~DownPersistence(); - - virtual Result initialize() override; - - virtual PartitionStateListResult getPartitionStates() const override; - - virtual BucketIdListResult listBuckets(PartitionId) const override; - - virtual Result setClusterState(const ClusterState&) override; - - virtual Result setActiveState(const Bucket&, - BucketInfo::ActiveState) override; - - virtual BucketInfoResult getBucketInfo(const Bucket&) const override; - - virtual Result put(const Bucket&, Timestamp, const Document::SP&, Context&) override; - - virtual RemoveResult remove(const Bucket&, - Timestamp timestamp, - const DocumentId& id, - Context&) override; - - virtual RemoveResult removeIfFound(const Bucket&, - Timestamp timestamp, - const DocumentId& id, - Context&) override; - - virtual Result removeEntry(const Bucket&, Timestamp, Context&) override; - - virtual UpdateResult update(const Bucket&, - Timestamp timestamp, - const DocumentUpdate::SP& update, - Context&) override; - - virtual Result flush(const Bucket&, Context&) override; - - virtual GetResult get(const Bucket&, - const document::FieldSet& fieldSet, - const DocumentId& id, - Context&) const override; - - virtual CreateIteratorResult createIterator( - const Bucket&, - const document::FieldSet& fieldSet, - const Selection& selection, //TODO: Make AST - IncludedVersions versions, - Context&) override; - - virtual IterateResult iterate(IteratorId id, - uint64_t maxByteSize, - Context&) const override; - - virtual Result destroyIterator(IteratorId id, Context&) override; - - virtual Result createBucket(const Bucket&, Context&) override; - - virtual Result deleteBucket(const Bucket&, Context&) override; - - virtual BucketIdListResult getModifiedBuckets() const override; - - virtual Result maintain(const Bucket&, - MaintenanceLevel level) override; - - virtual Result split(const Bucket& source, - const Bucket& target1, - const Bucket& target2, - Context&) override; - - virtual Result join(const Bucket& source1, - const Bucket& source2, - const Bucket& target, - Context&) override; - - virtual Result move(const Bucket&, PartitionId target, Context&) override; - + ~DownPersistence() override; + + Result initialize() override; + PartitionStateListResult getPartitionStates() const override; + BucketIdListResult listBuckets(PartitionId) const override; + Result setClusterState(const ClusterState&) override; + Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; + BucketInfoResult getBucketInfo(const Bucket&) const override; + Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; + RemoveResult remove(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&) override; + RemoveResult removeIfFound(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&) override; + Result removeEntry(const Bucket&, Timestamp, Context&) override; + UpdateResult update(const Bucket&, Timestamp timestamp, const DocumentUpdateSP& update, Context&) override; + Result flush(const Bucket&, Context&) override; + GetResult get(const Bucket&, const document::FieldSet& fieldSet, const DocumentId& id, Context&) const override; + + CreateIteratorResult createIterator(const Bucket&, const document::FieldSet& fieldSet, + const Selection& selection, IncludedVersions versions, Context&) override; + + IterateResult iterate(IteratorId id, uint64_t maxByteSize, Context&) const override; + Result destroyIterator(IteratorId id, Context&) override; + Result createBucket(const Bucket&, Context&) override; + Result deleteBucket(const Bucket&, Context&) override; + BucketIdListResult getModifiedBuckets() const override; + Result maintain(const Bucket&, MaintenanceLevel level) override; + Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; + Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; + Result move(const Bucket&, PartitionId target, Context&) override; }; } - -} diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index b24a94f0bea..57c4b62cddb 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -1,15 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/util/programoptions.h> -#include <string> -#include <iostream> + #include <vespa/searchcore/proton/server/proton.h> +#include <vespa/storage/storageserver/storagenode.h> #include <vespa/searchlib/util/statefile.h> #include <vespa/searchlib/util/sigbushandler.h> #include <vespa/searchlib/util/ioerrorhandler.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/metrics/metricmanager.h> +#include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/app.h> + +#include <string> +#include <iostream> + #include <vespa/log/log.h> LOG_SETUP("proton"); @@ -43,8 +47,7 @@ App::setupSignals() SIG::TERM.hook(); } -namespace -{ +namespace { vespalib::string getStateString(search::StateFile &stateFile) @@ -61,7 +64,6 @@ bool stateIsDown(const vespalib::string &stateString) return strstr(stateString.c_str(), "state=down") != nullptr; } - } Params @@ -102,9 +104,9 @@ public: PersistenceProvider *downPersistence); ~ProtonServiceLayerProcess() { shutdown(); } - virtual void shutdown() override; - virtual void setupProvider() override; - virtual storage::spi::PersistenceProvider& getProvider() override; + void shutdown() override; + void setupProvider() override; + storage::spi::PersistenceProvider& getProvider() override; void setMetricManager(metrics::MetricManager& mm) { // The service layer will call init(...) and stop() on the metric @@ -114,6 +116,7 @@ public: // down component. _metricManager = &mm; } + int64_t getGeneration() const override; }; ProtonServiceLayerProcess::ProtonServiceLayerProcess(const config::ConfigUri & @@ -151,6 +154,14 @@ ProtonServiceLayerProcess::getProvider() return _downPersistence ? *_downPersistence : _proton.getPersistence(); } +int64_t +ProtonServiceLayerProcess::getGeneration() const +{ + int64_t slGen = storage::ServiceLayerProcess::getGeneration(); + int64_t protonGen = _proton.getConfigGeneration(); + return std::min(slGen, protonGen); +} + int App::Main() { @@ -170,8 +181,7 @@ App::Main() if (proton.hasAbortedInit()) { EV_STOPPING("proton", "shutdown after aborted init"); } else { - const ProtonConfig &protonConfig = - configSnapshot->getProtonConfig(); + const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); vespalib::string basedir = protonConfig.basedir; bool stopOnIOErrors = protonConfig.stoponioerrors; vespalib::mkdir(basedir, true); @@ -184,19 +194,15 @@ App::Main() LOG(error, "proton state string is %s", stateString.c_str()); if (stopOnIOErrors) { if ( !params.serviceidentity.empty()) { - downPersistence.reset( - new DownPersistence("proton state string is " + - stateString)); + downPersistence.reset(new DownPersistence("proton state string is " + stateString)); } else { LOG(info, "Sleeping 900 seconds due to proton state"); int sleepLeft = 900; - while (!(SIG::INT.check() || SIG::TERM.check()) && - sleepLeft > 0) { + while (!(SIG::INT.check() || SIG::TERM.check()) && sleepLeft > 0) { FastOS_Thread::Sleep(1000); --sleepLeft; } - EV_STOPPING("proton", - "shutdown after stop on io errors"); + EV_STOPPING("proton", "shutdown after stop on io errors"); return 1; } } @@ -227,20 +233,14 @@ App::Main() stateGen = stateFile->getGen(); stateString = getStateString(*stateFile); if (stateIsDown(stateString)) { - LOG(error, "proton state string is %s", - stateString.c_str()); + LOG(error, "proton state string is %s", stateString.c_str()); if (stopOnIOErrors) { if (spiProton) { // report down state to cluster controller. - spiProton->getNode(). - notifyPartitionDown(0, - "proton state " - "string is " + - stateString); + spiProton->getNode().notifyPartitionDown(0, "proton state string is " + stateString); FastOS_Thread::Sleep(1000); } - EV_STOPPING("proton", - "shutdown after new stop on io errors"); + EV_STOPPING("proton", "shutdown after new stop on io errors"); return 1; } } diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index a0bb3cf8591..719abe08612 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -232,7 +232,7 @@ public: } void - close(void) + close() { for (DocumentDBMap::iterator itr = _docDbs.begin(); itr != _docDbs.end(); ++itr) { @@ -259,12 +259,12 @@ protected: } virtual - ~DocDBRepoHolder(void) + ~DocDBRepoHolder() { } void - close(void) + close() { if (_docDbRepo.get() != NULL) _docDbRepo->close(); @@ -320,7 +320,7 @@ public: } void - removeHandlers(void) + removeHandlers() { if (_docDbRepo.get() == NULL) return; @@ -332,7 +332,7 @@ public: } virtual - ~MyPersistenceEngine(void) + ~MyPersistenceEngine() { destroyIterators(); removeHandlers(); // Block calls to document db from engine @@ -385,12 +385,12 @@ public: struct FixtureBaseBase { - FixtureBaseBase(void) + FixtureBaseBase() { FastOS_FileInterface::EmptyAndRemoveDirectory("testdb"); } - ~FixtureBaseBase(void) + ~FixtureBaseBase() { FastOS_FileInterface::EmptyAndRemoveDirectory("testdb"); } @@ -408,7 +408,7 @@ struct FixtureBase : public FixtureBaseBase docType))) { } - ~FixtureBase(void) + ~FixtureBase() { } }; diff --git a/searchcore/src/apps/verify_ranksetup/.gitignore b/searchcore/src/apps/verify_ranksetup/.gitignore index 269b313f1f0..d32b68f0f38 100644 --- a/searchcore/src/apps/verify_ranksetup/.gitignore +++ b/searchcore/src/apps/verify_ranksetup/.gitignore @@ -1,4 +1,3 @@ .depend Makefile -verify_ranksetup -verify_ranksetup-bin +vespa-verify-ranksetup-bin diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt index de777f88182..7dd6d0490da 100644 --- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(searchcore_verify_ranksetup_app SOURCES verify_ranksetup.cpp - OUTPUT_NAME verify_ranksetup-bin + OUTPUT_NAME vespa-verify-ranksetup-bin INSTALL bin DEPENDS searchcore_fconfig diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 87c85f99752..5731700a55f 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("verify_ranksetup"); #include <vespa/config-attributes.h> #include <vespa/config-imported-fields.h> @@ -19,6 +16,10 @@ LOG_SETUP("verify_ranksetup"); #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/plugin/setup.h> +#include <vespa/fastos/app.h> + +#include <vespa/log/log.h> +LOG_SETUP("verify_ranksetup"); using config::ConfigContext; using config::ConfigHandle; diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 7b7527da6ef..7966a705802 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -1,17 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("vespa-gen-testdocs"); -#include <algorithm> -#include <string> + #include <vespa/searchlib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/fastos/app.h> #include <iostream> +#include <algorithm> +#include <sstream> #include <openssl/sha.h> +#include <vespa/log/log.h> +LOG_SETUP("vespa-gen-testdocs"); + typedef vespalib::hash_set<vespalib::string> StringSet; typedef vespalib::hash_set<uint32_t> UIntSet; typedef std::vector<vespalib::string> StringArray; @@ -20,7 +22,7 @@ using namespace vespalib::alloc; using vespalib::string; void -usageHeader(void) +usageHeader() { using std::cerr; cerr << @@ -148,13 +150,13 @@ public: FieldGenerator(const string &name); virtual - ~FieldGenerator(void); + ~FieldGenerator(); virtual void - setup(void); + setup(); virtual void - clear(void); + clear(); virtual void deleteHistogram(const string &baseDir, @@ -175,19 +177,19 @@ FieldGenerator::FieldGenerator(const string &name) { } -FieldGenerator::~FieldGenerator(void) +FieldGenerator::~FieldGenerator() { } void -FieldGenerator::setup(void) +FieldGenerator::setup() { } void -FieldGenerator::clear(void) +FieldGenerator::clear() { } @@ -229,13 +231,13 @@ public: uint32_t maxFill); virtual - ~RandTextFieldGenerator(void); + ~RandTextFieldGenerator(); virtual void - setup(void) override; + setup() override; virtual void - clear(void) override; + clear() override; virtual void deleteHistogram(const string &baseDir, const string &name) override; @@ -267,13 +269,13 @@ RandTextFieldGenerator::RandTextFieldGenerator(const string &name, -RandTextFieldGenerator::~RandTextFieldGenerator(void) +RandTextFieldGenerator::~RandTextFieldGenerator() { } void -RandTextFieldGenerator::setup(void) +RandTextFieldGenerator::setup() { LOG(info, "generating dictionary for field %s (%u words)", @@ -284,7 +286,7 @@ RandTextFieldGenerator::setup(void) void -RandTextFieldGenerator::clear(void) +RandTextFieldGenerator::clear() { typedef std::vector<uint32_t>::iterator HI; for (HI i(_histogram.begin()), ie(_histogram.end()); i != ie; ++i) { @@ -363,10 +365,10 @@ public: const std::vector<uint32_t> &mods); virtual - ~ModTextFieldGenerator(void); + ~ModTextFieldGenerator(); virtual void - clear(void) override; + clear() override; virtual void writeHistogram(const string &name); @@ -386,13 +388,13 @@ ModTextFieldGenerator::ModTextFieldGenerator(const string &name, } -ModTextFieldGenerator::~ModTextFieldGenerator(void) +ModTextFieldGenerator::~ModTextFieldGenerator() { } void -ModTextFieldGenerator::clear(void) +ModTextFieldGenerator::clear() { } @@ -427,10 +429,10 @@ public: IdTextFieldGenerator(const string &name); virtual - ~IdTextFieldGenerator(void); + ~IdTextFieldGenerator(); virtual void - clear(void) override; + clear() override; virtual void writeHistogram(const string &name); @@ -446,13 +448,13 @@ IdTextFieldGenerator::IdTextFieldGenerator(const string &name) } -IdTextFieldGenerator::~IdTextFieldGenerator(void) +IdTextFieldGenerator::~IdTextFieldGenerator() { } void -IdTextFieldGenerator::clear(void) +IdTextFieldGenerator::clear() { } @@ -486,10 +488,10 @@ public: uint32_t count); virtual - ~RandIntFieldGenerator(void); + ~RandIntFieldGenerator(); virtual void - clear(void) override; + clear() override; virtual void writeHistogram(const string &name); @@ -512,13 +514,13 @@ RandIntFieldGenerator::RandIntFieldGenerator(const string &name, } -RandIntFieldGenerator::~RandIntFieldGenerator(void) +RandIntFieldGenerator::~RandIntFieldGenerator() { } void -RandIntFieldGenerator::clear(void) +RandIntFieldGenerator::clear() { } @@ -549,16 +551,16 @@ class DocumentGenerator const FieldVec _fields; void - setup(void); + setup(); public: DocumentGenerator(const string &docType, const string &idPrefix, const FieldVec &fields); - ~DocumentGenerator(void); + ~DocumentGenerator(); void - clear(void); + clear(); void deleteHistogram(const string &baseDir, @@ -591,12 +593,12 @@ DocumentGenerator::DocumentGenerator(const string &docType, } -DocumentGenerator::~DocumentGenerator(void) +DocumentGenerator::~DocumentGenerator() { } void -DocumentGenerator::setup(void) +DocumentGenerator::setup() { typedef FieldVec::const_iterator FI; for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { @@ -606,7 +608,7 @@ DocumentGenerator::setup(void) void -DocumentGenerator::clear(void) +DocumentGenerator::clear() { typedef FieldVec::const_iterator FI; for (FI i(_fields.begin()), ie(_fields.end()); i != ie; ++i) { @@ -694,7 +696,7 @@ public: } virtual - ~SubApp(void) + ~SubApp() { } @@ -702,10 +704,10 @@ public: usage(bool showHeader) = 0; virtual bool - getOptions(void) = 0; + getOptions() = 0; virtual int - run(void) = 0; + run() = 0; }; class GenTestDocsApp : public SubApp @@ -748,7 +750,7 @@ public: } virtual - ~GenTestDocsApp(void) + ~GenTestDocsApp() { } @@ -756,10 +758,10 @@ public: usage(bool showHeader) override; virtual bool - getOptions(void) override; + getOptions() override; virtual int - run(void) override; + run() override; }; @@ -785,7 +787,7 @@ GenTestDocsApp::usage(bool showHeader) } bool -GenTestDocsApp::getOptions(void) +GenTestDocsApp::getOptions() { int c; const char *optArgument = NULL; @@ -898,7 +900,7 @@ GenTestDocsApp::getOptions(void) int -GenTestDocsApp::run(void) +GenTestDocsApp::run() { printf("Hello world\n"); string idPrefix("id:test:"); @@ -918,21 +920,21 @@ class App : public FastOS_Application { public: void - usage(void); + usage(); int - Main(void) override; + Main() override; }; void -App::usage(void) +App::usage() { GenTestDocsApp(*this).usage(true); } int -App::Main(void) +App::Main() { if (_argc < 2) { usage(); diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp index 3931daf7fdd..58844dc969c 100644 --- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp +++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp @@ -1,18 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("vespa-proton-cmd"); -#include <vespa/fnet/frt/frt.h> -#include <algorithm> -#include <string> + #include <vespa/slobrok/sbmirror.h> #include <vespa/config-slobroks.h> -#include <vespa/vespalib/util/host_name.h> #include <vespa/config/common/configsystem.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/util/host_name.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fastos/app.h> +#include <algorithm> +#include <string> +#include <vespa/log/log.h> +LOG_SETUP("vespa-proton-cmd"); -namespace pandora { -namespace rtc_cmd { +namespace pandora::rtc_cmd { class App : public FastOS_Application { @@ -39,11 +40,8 @@ public: { fprintf(stderr, "usage: %s <port|spec|--local|--id=name> <cmd> [args]\n", _argv[0]); fprintf(stderr, "die\n"); - fprintf(stderr, "getConfigTime\n"); fprintf(stderr, "getProtonStatus\n"); fprintf(stderr, "getState\n"); - fprintf(stderr, "listDocTypes\n"); - fprintf(stderr, "listSchema documentType\n"); fprintf(stderr, "monitor\n"); fprintf(stderr, "enableSearching\n"); fprintf(stderr, "disableSearching\n"); @@ -87,7 +85,7 @@ public: } void - monitorLoop(void); + monitorLoop(); void scanSpecs(slobrok::api::MirrorAPI::SpecList &specs, @@ -337,71 +335,6 @@ public: if (! _req->IsError()) { printf("OK: prepareRestart enabled\n"); } - } else if (strcmp(_argv[2], "listDocTypes") == 0) { - _req->SetMethodName("proton.listDocTypes"); - invokeRPC(false, 86400.0); - invoked = true; - if (! _req->IsError()) { - FRT_Values &ret = *_req->GetReturn(); - if (strcmp(ret.GetTypeString(), "S") == 0) { - uint32_t dtLen = ret[0]._string_array._len; - const FRT_StringValue *dt = ret[0]._string_array._pt; - for (uint32_t i = 0; i < dtLen; ++i) { - if (i > 0) - printf(" "); - printf("%s", dt[i]._str); - } - printf("\n"); - } else { - fprintf(stderr, "Unexpected return value\n"); - } - } - } else if (strcmp(_argv[2], "listSchema") == 0 && _argc == 4) { - _req->SetMethodName("proton.listSchema"); - FRT_Values &arg = *_req->GetParams(); - arg.AddString(_argv[3]); - invokeRPC(false, 86400.0); - invoked = true; - if (! _req->IsError()) { - FRT_Values &ret = *_req->GetReturn(); - if (strcmp(ret.GetTypeString(), "SSSS") == 0) { - uint32_t fnLen = ret[0]._string_array._len; - const FRT_StringValue *fn = ret[0]._string_array._pt; - uint32_t fdtLen = ret[1]._string_array._len; - const FRT_StringValue *fdt = ret[1]._string_array._pt; - uint32_t fctLen = ret[2]._string_array._len; - const FRT_StringValue *fct = ret[2]._string_array._pt; - uint32_t flLen = ret[3]._string_array._len; - const FRT_StringValue *fl = ret[3]._string_array._pt; - for (uint32_t i = 0; - i < fnLen && i < fdtLen && i < fctLen && i < flLen; - ++i) { - if (i > 0) - printf(" "); - printf("%s/%s/%s/%s", - fn[i]._str, - fdt[i]._str, - fct[i]._str, - fl[i]._str); - } - printf("\n"); - } else { - fprintf(stderr, "Unexpected return value\n"); - } - } - } else if (strcmp(_argv[2], "getConfigTime") == 0) { - _req->SetMethodName("proton.getConfigTime"); - invokeRPC(false, 86400.0); - invoked = true; - if (! _req->IsError()) { - FRT_Values &ret = *_req->GetReturn(); - if (strcmp(ret.GetTypeString(), "l") == 0) { - uint64_t configTime = ret[0]._intval64; - printf("%" PRId64 "\n", configTime); - } else { - fprintf(stderr, "Unexpected return value\n"); - } - } } else if (strcmp(_argv[2], "die") == 0) { _req->SetMethodName("pandora.rtc.die"); @@ -421,7 +354,7 @@ public: void -App::monitorLoop(void) +App::monitorLoop() { for (;;) { FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); @@ -473,7 +406,6 @@ App::monitorLoop(void) } } // namespace pandora::rtc_cmd -} // namespace pandora int main(int argc, char **argv) diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 73e1fcd09a5..0fdfb09ecef 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/util/xmlstream.h> #include <vespa/document/config/config-documenttypes.h> #include <iostream> #include <vespa/config/helper/configgetter.hpp> @@ -47,10 +48,10 @@ class ConfigFile std::vector<char> _content; public: - ConfigFile(void); + ConfigFile(); const vespalib::string & - getName(void) const + getName() const { return _name; } @@ -59,11 +60,11 @@ public: deserialize(vespalib::nbostream &stream); void - print(void) const; + print() const; }; -ConfigFile::ConfigFile(void) +ConfigFile::ConfigFile() : _name(), _modTime(0), _content() @@ -87,7 +88,7 @@ ConfigFile::deserialize(vespalib::nbostream &stream) } void -ConfigFile::print(void) const +ConfigFile::print() const { std::cout << "Name: " << _name << "\n" << "ModTime: " << _modTime << "\n" << @@ -108,7 +109,7 @@ operator>>(vespalib::nbostream &stream, ConfigFile &configFile) struct DummyStreamHandler : public NewConfigOperation::IStreamHandler { std::map<std::string, ConfigFile> _cfs; - DummyStreamHandler(void) + DummyStreamHandler() : NewConfigOperation::IStreamHandler(), _cfs() { diff --git a/searchcore/src/testlist.txt b/searchcore/src/testlist.txt index bbe72775946..b74cd851052 100644 --- a/searchcore/src/testlist.txt +++ b/searchcore/src/testlist.txt @@ -59,7 +59,6 @@ tests/proton/reprocessing/attribute_reprocessing_initializer tests/proton/reprocessing/document_reprocessing_handler tests/proton/reprocessing/reprocessing_runner tests/proton/server -tests/proton/server/data_directory_upgrader tests/proton/server/disk_mem_usage_filter tests/proton/server/health_adapter tests/proton/server/memoryflush diff --git a/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp b/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp index 375afee1777..0341cbb17a9 100644 --- a/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp +++ b/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("randomrow_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp b/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp index 8dd3ada4270..bec260038e8 100644 --- a/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp +++ b/searchcore/src/tests/fdispatch/search_path/search_path_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("search_path_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/attribute/.gitignore b/searchcore/src/tests/proton/attribute/.gitignore index 794f5f454f8..46973062589 100644 --- a/searchcore/src/tests/proton/attribute/.gitignore +++ b/searchcore/src/tests/proton/attribute/.gitignore @@ -4,6 +4,5 @@ Makefile test test_output flush - searchcore_attribute_test_app searchcore_attributeflush_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index d8407355e41..21c965b7908 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/test/attribute_utils.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp index 9d1ab5e3fef..471fcc5c7e1 100644 --- a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp @@ -1,16 +1,19 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/attributedisklayout.h> -#include <vespa/searchcore/proton/test/directory_handler.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/log/log.h> LOG_SETUP("attribute_directory_test"); using search::IndexMetaInfo; using search::SerialNum; +using search::test::DirectoryHandler; namespace proton { @@ -44,13 +47,13 @@ bool hasWriter(const std::unique_ptr<AttributeDirectory::Writer> &writer) { } -struct Fixture : public test::DirectoryHandler +struct Fixture : public DirectoryHandler { std::shared_ptr<AttributeDiskLayout> _diskLayout; Fixture() - : test::DirectoryHandler("attributes"), + : DirectoryHandler("attributes"), _diskLayout(AttributeDiskLayout::create("attributes")) { } diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index f5eb55f2abc..74320164965 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -1,19 +1,20 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchcore/proton/test/directory_handler.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchcore/proton/test/attribute_utils.h> -#include <vespa/searchcore/proton/attribute/attributedisklayout.h> #include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/attribute_factory.h> #include <vespa/searchcore/proton/attribute/attribute_initializer.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/stllike/string.h> using search::attribute::Config; using search::attribute::BasicType; using search::attribute::CollectionType; using search::SerialNum; +using search::test::DirectoryHandler; const vespalib::string test_dir = "test_output"; @@ -69,7 +70,7 @@ saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, S struct Fixture { - test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; std::shared_ptr<AttributeDiskLayout> _diskLayout; AttributeFactory _factory; Fixture(); diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 6bf9544b3ea..78c4425fdb1 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_manager_test"); #include <vespa/config-attributes.h> #include <vespa/fastos/file.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> @@ -14,29 +14,30 @@ LOG_SETUP("attribute_manager_test"); #include <vespa/searchcore/proton/attribute/i_attribute_functor.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/initializer/task_runner.h> +#include <vespa/searchcore/proton/server/executor_thread_service.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcore/proton/test/attribute_vectors.h> -#include <vespa/searchcore/proton/test/directory_handler.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/common/foregroundtaskexecutor.h> #include <vespa/searchlib/common/indexmetainfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/searchlib/attribute/reference_attribute.h> -#include <vespa/searchcommon/attribute/iattributevector.h> namespace vespa { namespace config { namespace search {}}} @@ -60,6 +61,7 @@ using search::attribute::ReferenceAttribute; using search::index::DummyFileHeaderContext; using search::predicate::PredicateIndex; using search::predicate::PredicateTreeAnnotations; +using search::test::DirectoryHandler; using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespalib::eval::ValueType; @@ -103,13 +105,13 @@ const AVConfig INT32_ARRAY = AttributeUtils::getInt32ArrayConfig(); void fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { - test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); + AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); } void fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { - test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); + AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); } search::SerialNum getCreateSerialNum(const AttributeGuard::UP &guard) @@ -141,7 +143,7 @@ struct ImportedAttributesRepoBuilder { struct BaseFixture { - test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; @@ -223,6 +225,8 @@ struct ParallelAttributeManager size_t attributeGrowNumDocs; bool fastAccessAttributesOnly; std::shared_ptr<AttributeManager::SP> mgr; + vespalib::ThreadStackExecutor masterExecutor; + ExecutorThreadService master; AttributeManagerInitializer::SP initializer; ParallelAttributeManager(search::SerialNum configSerialNum, AttributeManager::SP baseAttrMgr, @@ -239,10 +243,12 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria attributeGrowNumDocs(1), fastAccessAttributesOnly(false), mgr(std::make_shared<AttributeManager::SP>()), + masterExecutor(1, 128 * 1024), + master(masterExecutor), initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg, attributeGrow, attributeGrowNumDocs, - fastAccessAttributesOnly, mgr)) + fastAccessAttributesOnly, master, mgr)) { documentMetaStore->setCommittedDocIdLimit(docIdLimit); vespalib::ThreadStackExecutor executor(3, 128 * 1024); @@ -795,6 +801,45 @@ TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) TEST_DO(assertCreateSerialNum(am2.mgr, "a6", 20)); } +void assertShrinkTargetSerial(proton::AttributeManager &mgr, const vespalib::string &name, search::SerialNum expSerialNum) +{ + auto shrinker = mgr.getShrinker(name); + EXPECT_EQUAL(expSerialNum, shrinker->getFlushedSerialNum()); +} + +TEST_F("require that we can guess flushed serial number for shrink flushtarget", BaseFixture) +{ + auto am1(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + am1->addAttribute({"a1", INT32_SINGLE}, 1); + am1->addAttribute({"a2", INT32_SINGLE}, 2); + TEST_DO(assertShrinkTargetSerial(*am1, "a1", 0)); + TEST_DO(assertShrinkTargetSerial(*am1, "a2", 1)); + am1->flushAll(10); + am1 = std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo); + am1->addAttribute({"a1", INT32_SINGLE}, 1); + am1->addAttribute({"a2", INT32_SINGLE}, 2); + TEST_DO(assertShrinkTargetSerial(*am1, "a1", 10)); + TEST_DO(assertShrinkTargetSerial(*am1, "a2", 10)); +} + +TEST_F("require that shrink flushtarget is handed over to new attribute manager", BaseFixture) +{ + auto am1(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + am1->addAttribute({"a1", INT32_SINGLE}, 4); + AttrSpecList newSpec; + newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); + auto am2 = am1->create(AttrMgrSpec(newSpec, 5, 20)); + auto am3 = std::dynamic_pointer_cast<AttributeManager>(am2); + TEST_DO(assertShrinkTargetSerial(*am3, "a1", 3)); + EXPECT_EQUAL(am1->getShrinker("a1"), am3->getShrinker("a1")); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index 41ff853047d..3227521e8e7 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -1,26 +1,28 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_populator_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/common/schema.h> -#include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/attribute_populator.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/test.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchcore/proton/common/hw_info.h> -#include <vespa/searchlib/common/foregroundtaskexecutor.h> using namespace document; using namespace proton; using namespace search; using namespace search::index; -typedef search::attribute::Config AVConfig; -typedef search::attribute::BasicType AVBasicType; +using search::test::DirectoryHandler; + +using AVBasicType = search::attribute::BasicType; +using AVConfig = search::attribute::Config; const vespalib::string TEST_DIR = "testdir"; const uint64_t CREATE_SERIAL_NUM = 8u; @@ -53,7 +55,7 @@ struct DocContext struct Fixture { - test::DirectoryHandler _testDir; + DirectoryHandler _testDir; DummyFileHeaderContext _fileHeader; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; @@ -100,5 +102,6 @@ TEST_F("require that reprocess with document populates attribute", Fixture) TEST_MAIN() { + vespalib::rmdir(TEST_DIR, true); TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 173d8dd2a2a..c60c5f3a622 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); @@ -13,6 +12,7 @@ LOG_SETUP("attribute_test"); #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/common/idestructorcallback.h> @@ -32,7 +32,7 @@ LOG_SETUP("attribute_test"); #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/searchlib/common/foregroundtaskexecutor.h> #include <vespa/searchlib/common/sequencedtaskexecutorobserver.h> -#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/types.h> #include <vespa/eval/tensor/default_tensor.h> @@ -50,12 +50,14 @@ using namespace search::index; using namespace search; using namespace vespa::config::search; +using proton::test::AttributeUtils; using search::TuneFileAttributes; using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; using search::predicate::PredicateHash; using search::predicate::PredicateIndex; using search::tensor::TensorAttribute; +using search::test::DirectoryHandler; using std::string; using vespalib::eval::ValueType; using vespalib::tensor::Tensor; @@ -88,13 +90,13 @@ const AVConfig INT32_ARRAY = unregister(AVConfig(AVBasicType::INT32, AVCollectio void fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { - test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); + AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); } void fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { - test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); + AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); } const std::shared_ptr<IDestructorCallback> emptyCallback; @@ -102,7 +104,7 @@ const std::shared_ptr<IDestructorCallback> emptyCallback; struct Fixture { - test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriterReal; SequencedTaskExecutorObserver _attributeFieldWriter; @@ -518,7 +520,7 @@ const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; struct FilterFixture { - test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; @@ -550,6 +552,10 @@ TEST_F("require that filter attribute manager can filter attributes", FilterFixt f._filterMgr.getAttributeList(attrs); EXPECT_EQUAL(1u, attrs.size()); EXPECT_EQUAL("a2", attrs[0]->getName()); + searchcorespi::IFlushTarget::List targets = f._filterMgr.getFlushTargets(); + EXPECT_EQUAL(2u, targets.size()); + EXPECT_EQUAL("attribute.flush.a2", targets[0]->getName()); + EXPECT_EQUAL("attribute.shrink.a2", targets[1]->getName()); } TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 808930a7e8b..20497f262cb 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_usage_filter_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index 5375f7817ec..a3b5d6dd2fb 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -1,19 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/searchcore/proton/attribute/attributemanager.h> -#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> #include <vespa/searchlib/common/indexmetainfo.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/common/foregroundtaskexecutor.h> -#include <vespa/searchcore/proton/test/directory_handler.h> -#include <vespa/fastos/file.h> +#include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> LOG_SETUP("attributeflush_test"); @@ -53,7 +54,7 @@ public: } virtual void - run(void) override + run() override { _task->run(); _gate->countDown(); @@ -103,7 +104,7 @@ public: startFlushing(uint64_t syncToken, FlushHandler & handler); void - run(void); + run(); }; @@ -118,7 +119,7 @@ UpdaterTask::startFlushing(uint64_t syncToken, FlushHandler & handler) void -UpdaterTask::run(void) +UpdaterTask::run() { LOG(info, "UpdaterTask::run(begin)"); uint32_t totalDocs = 2000000; @@ -194,28 +195,28 @@ class Test : public vespalib::TestApp { private: void - requireThatUpdaterAndFlusherCanRunConcurrently(void); + requireThatUpdaterAndFlusherCanRunConcurrently(); void - requireThatFlushableAttributeReportsMemoryUsage(void); + requireThatFlushableAttributeReportsMemoryUsage(); void requireThatFlushableAttributeManagesSyncTokenInfo(); void - requireThatFlushTargetsCanBeRetrieved(void); + requireThatFlushTargetsCanBeRetrieved(); void - requireThatCleanUpIsPerformedAfterFlush(void); + requireThatCleanUpIsPerformedAfterFlush(); void - requireThatFlushStatsAreUpdated(void); + requireThatFlushStatsAreUpdated(); void - requireThatOnlyOneFlusherCanRunAtTheSameTime(void); + requireThatOnlyOneFlusherCanRunAtTheSameTime(); void - requireThatLastFlushTimeIsReported(void); + requireThatLastFlushTimeIsReported(); void requireThatShrinkWorks(); @@ -224,7 +225,7 @@ private: void requireThatFlushedAttributeCanBeLoaded(); public: int - Main(void) override; + Main() override; }; @@ -297,7 +298,7 @@ struct Fixture : public BaseFixture, public AttributeManagerFixture void -Test::requireThatUpdaterAndFlusherCanRunConcurrently(void) +Test::requireThatUpdaterAndFlusherCanRunConcurrently() { Fixture f; AttributeManager &am = f._m; @@ -329,7 +330,7 @@ Test::requireThatUpdaterAndFlusherCanRunConcurrently(void) void -Test::requireThatFlushableAttributeReportsMemoryUsage(void) +Test::requireThatFlushableAttributeReportsMemoryUsage() { Fixture f; AttributeManager &am = f._m; @@ -382,21 +383,23 @@ Test::requireThatFlushableAttributeManagesSyncTokenInfo() void -Test::requireThatFlushTargetsCanBeRetrieved(void) +Test::requireThatFlushTargetsCanBeRetrieved() { Fixture f; AttributeManager &am = f._m; f.addAttribute("a4"); f.addAttribute("a5"); std::vector<IFlushTarget::SP> ftl = am.getFlushTargets(); - EXPECT_EQUAL(2u, ftl.size()); + EXPECT_EQUAL(4u, ftl.size()); EXPECT_EQUAL(am.getFlushable("a4").get(), ftl[0].get()); - EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[1].get()); + EXPECT_EQUAL(am.getShrinker("a4").get(), ftl[1].get()); + EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[2].get()); + EXPECT_EQUAL(am.getShrinker("a5").get(), ftl[3].get()); } void -Test::requireThatCleanUpIsPerformedAfterFlush(void) +Test::requireThatCleanUpIsPerformedAfterFlush() { Fixture f; AttributeVector::SP av = f.addAttribute("a6"); @@ -432,7 +435,7 @@ Test::requireThatCleanUpIsPerformedAfterFlush(void) void -Test::requireThatFlushStatsAreUpdated(void) +Test::requireThatFlushStatsAreUpdated() { Fixture f; AttributeManager &am = f._m; @@ -448,7 +451,7 @@ Test::requireThatFlushStatsAreUpdated(void) void -Test::requireThatOnlyOneFlusherCanRunAtTheSameTime(void) +Test::requireThatOnlyOneFlusherCanRunAtTheSameTime() { Fixture f; AttributeManager &am = f._m; @@ -481,7 +484,7 @@ Test::requireThatOnlyOneFlusherCanRunAtTheSameTime(void) void -Test::requireThatLastFlushTimeIsReported(void) +Test::requireThatLastFlushTimeIsReported() { BaseFixture f; FastOS_StatInfo stat; @@ -527,8 +530,8 @@ Test::requireThatShrinkWorks() AttributeVector::SP av = f.addAttribute("a10"); av->addDocs(1000 - av->getNumDocs()); - av->commit(10, 10); - IFlushTarget::SP ft = am.getFlushable("a10"); + av->commit(50, 50); + IFlushTarget::SP ft = am.getShrinker("a10"); EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), ft->getApproxMemoryGain().getAfter()); AttributeGuard::UP g = am.getAttribute("a10"); @@ -541,21 +544,22 @@ Test::requireThatShrinkWorks() EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(1000u, av->getNumDocs()); EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); - f._aw.heartBeat(11); + f._aw.heartBeat(51); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_FALSE(av->canShrinkLidSpace()); EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), ft->getApproxMemoryGain().getAfter()); g.reset(); - f._aw.heartBeat(11); + f._aw.heartBeat(52); EXPECT_TRUE(av->wantShrinkLidSpace()); EXPECT_TRUE(av->canShrinkLidSpace()); EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() > ft->getApproxMemoryGain().getAfter()); EXPECT_EQUAL(1000u, av->getNumDocs()); EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); + EXPECT_EQUAL(createSerialNum - 1, ft->getFlushedSerialNum()); vespalib::ThreadStackExecutor exec(1, 128 * 1024); - vespalib::Executor::Task::UP task = ft->initFlush(11); + vespalib::Executor::Task::UP task = ft->initFlush(53); exec.execute(std::move(task)); exec.sync(); exec.shutdown(); @@ -606,7 +610,7 @@ Test::requireThatFlushedAttributeCanBeLoaded() } int -Test::Main(void) +Test::Main() { TEST_INIT("attributeflush_test"); diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index 49def82ec59..2783988feca 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -1,24 +1,24 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attributes_state_explorer_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/attribute_vectors.h> -#include <vespa/searchcore/proton/test/directory_handler.h> -#include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/searchcore/proton/common/hw_info.h> using namespace proton; using namespace proton::test; -using search::index::DummyFileHeaderContext; using search::AttributeVector; -using search::TuneFileAttributes; using search::ForegroundTaskExecutor; +using search::TuneFileAttributes; +using search::index::DummyFileHeaderContext; +using search::test::DirectoryHandler; const vespalib::string TEST_DIR = "test_output"; diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp index 8e6d9050d88..6b8be342609 100644 --- a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("document_field_populator_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp index 7cb6a503ae8..c10dae9f14e 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index eafe4f00639..23b9f464f8c 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("imported_attributes_context_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp index d9b4031e22f..f7b4d32ac56 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_repo/imported_attributes_repo_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("imported_attributes_repo_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp index b5fa3ae044e..246dee98fba 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("bucketdb_test"); diff --git a/searchcore/src/tests/proton/common/.gitignore b/searchcore/src/tests/proton/common/.gitignore index c03144e885d..9ce51ef2178 100644 --- a/searchcore/src/tests/proton/common/.gitignore +++ b/searchcore/src/tests/proton/common/.gitignore @@ -1,3 +1,2 @@ searchcore_cachedselect_test_app -searchcore_schemautil_test_app searchcore_selectpruner_test_app diff --git a/searchcore/src/tests/proton/common/CMakeLists.txt b/searchcore/src/tests/proton/common/CMakeLists.txt index 5f0fa227c88..73f300fd124 100644 --- a/searchcore/src/tests/proton/common/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/CMakeLists.txt @@ -1,11 +1,4 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_schemautil_test_app TEST - SOURCES - schemautil_test.cpp - DEPENDS - searchcore_pcommon -) -vespa_add_test(NAME searchcore_schemautil_test_app COMMAND searchcore_schemautil_test_app) vespa_add_executable(searchcore_selectpruner_test_app TEST SOURCES selectpruner_test.cpp diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp index 6d1f7735479..1117234f18f 100644 --- a/searchcore/src/tests/proton/common/cachedselect_test.cpp +++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp @@ -23,6 +23,7 @@ #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/log/log.h> LOG_SETUP("cachedselect_test"); @@ -88,7 +89,7 @@ const int32_t noIntVal = std::numeric_limits<int32_t>::min(); DocumentTypeRepo::UP -makeDocTypeRepo(void) +makeDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, @@ -194,7 +195,7 @@ public: } uint32_t - getGets(void) const + getGets() const { return _gets; } @@ -293,9 +294,9 @@ public: MyAttributeManager _amgr; MyDB::UP _db; - TestFixture(void); + TestFixture(); - ~TestFixture(void); + ~TestFixture(); CachedSelect::SP testParse(const string &selection, @@ -304,7 +305,7 @@ public: }; -TestFixture::TestFixture(void) +TestFixture::TestFixture() : _repoUP(), _hasFields(true), _amgr(), @@ -320,7 +321,7 @@ TestFixture::TestFixture(void) } -TestFixture::~TestFixture(void) +TestFixture::~TestFixture() { } diff --git a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp index e323ef6fe8f..c42a8ba9a7f 100644 --- a/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp +++ b/searchcore/src/tests/proton/common/document_type_inspector/document_type_inspector_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/configbuilder.h> @@ -16,7 +15,7 @@ const vespalib::string header_name = type_name + ".header"; const vespalib::string body_name = type_name + ".body"; DocumentTypeRepo::UP -makeOldDocTypeRepo(void) +makeOldDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, @@ -29,7 +28,7 @@ makeOldDocTypeRepo(void) } DocumentTypeRepo::UP -makeNewDocTypeRepo(void) +makeNewDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, diff --git a/searchcore/src/tests/proton/common/dummydbowner.h b/searchcore/src/tests/proton/common/dummydbowner.h index 1b0b609e33d..9a9adde19eb 100644 --- a/searchcore/src/tests/proton/common/dummydbowner.h +++ b/searchcore/src/tests/proton/common/dummydbowner.h @@ -4,27 +4,20 @@ #include <vespa/searchcore/proton/server/idocumentdbowner.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> #include <vespa/vespalib/stllike/string.h> -namespace proton -{ +namespace proton { struct DummyDBOwner : IDocumentDBOwner { std::shared_ptr<IDocumentDBReferenceRegistry> _registry; DummyDBOwner() : _registry(std::make_shared<DocumentDBReferenceRegistry>()) - { - } + {} ~DummyDBOwner() {} bool isInitializing() const override { return false; } - searchcorespi::IIndexManagerFactory::SP - getIndexManagerFactory(const vespalib::stringref & ) const override { - return searchcorespi::IIndexManagerFactory::SP(); - } uint32_t getDistributionKey() const override { return -1; } std::shared_ptr<IDocumentDBReferenceRegistry> getDocumentDBReferenceRegistry() const override { return _registry; @@ -32,4 +25,3 @@ struct DummyDBOwner : IDocumentDBOwner { }; } // namespace proton - diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp index e9ea623d554..0090372635d 100644 --- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchcore/proton/common/hw_info_sampler.h> +#include <vespa/config/print/fileconfigwriter.h> #include <vespa/searchcore/config/config-hwinfo.h> -#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchcore/proton/common/hw_info_sampler.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/config/print/fileconfigwriter.h> using proton::HwInfoSampler; +using search::test::DirectoryHandler; using vespa::config::search::core::HwinfoConfig; using vespa::config::search::core::HwinfoConfigBuilder; @@ -28,7 +28,7 @@ long time_point_to_long(Clock::time_point tp) struct Fixture { - proton::test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; Fixture() : _dirHandler(test_dir) diff --git a/searchcore/src/tests/proton/common/schemautil_test.cpp b/searchcore/src/tests/proton/common/schemautil_test.cpp deleted file mode 100644 index d21582c5111..00000000000 --- a/searchcore/src/tests/proton/common/schemautil_test.cpp +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for schemautil. - -#include <vespa/fastos/fastos.h> -#include <vespa/searchcore/proton/common/schemautil.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> - -using namespace proton; -using namespace search::index; -using search::index::schema::CollectionType; -using search::index::schema::DataType; -using vespalib::string; - - -namespace { - -void addAllFieldTypes(const string &name, Schema &schema, - fastos::TimeStamp timestamp) -{ - Schema::IndexField index_field(name, DataType::STRING); - index_field.setTimestamp(timestamp); - schema.addIndexField(index_field); - - Schema::AttributeField attribute_field(name, DataType::STRING); - attribute_field.setTimestamp(timestamp); - schema.addAttributeField(attribute_field); - - Schema::SummaryField summary_field(name, DataType::STRING); - summary_field.setTimestamp(timestamp); - schema.addSummaryField(summary_field); -} - -TEST("require that makeHistorySchema sets timestamp") -{ - Schema old_schema; - Schema new_schema; - Schema old_history; - - const fastos::TimeStamp now(84); - const string name = "foo"; - addAllFieldTypes(name, old_schema, fastos::TimeStamp(0)); - - Schema::SP schema = SchemaUtil::makeHistorySchema(new_schema, old_schema, - old_history, now); - - ASSERT_EQUAL(1u, schema->getNumIndexFields()); - EXPECT_EQUAL(name, schema->getIndexField(0).getName()); - EXPECT_EQUAL(now, schema->getIndexField(0).getTimestamp()); - - ASSERT_EQUAL(1u, schema->getNumAttributeFields()); - EXPECT_EQUAL(name, schema->getAttributeField(0).getName()); - EXPECT_EQUAL(now, schema->getAttributeField(0).getTimestamp()); - - ASSERT_EQUAL(1u, schema->getNumSummaryFields()); - EXPECT_EQUAL(name, schema->getSummaryField(0).getName()); - EXPECT_EQUAL(now, schema->getSummaryField(0).getTimestamp()); -} - -TEST("require that makeHistorySchema preserves timestamp") -{ - Schema old_schema; - Schema new_schema; - Schema old_history; - - const fastos::TimeStamp timestamp(42); - const string name = "foo"; - addAllFieldTypes("bar", old_schema, fastos::TimeStamp(0)); - addAllFieldTypes(name, old_history, timestamp); - - Schema::SP schema = - SchemaUtil::makeHistorySchema(new_schema, old_schema, old_history); - - ASSERT_EQUAL(2u, schema->getNumIndexFields()); - uint32_t id = schema->getIndexFieldId(name); - ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID); - EXPECT_EQUAL(timestamp, schema->getIndexField(id).getTimestamp()); - - ASSERT_EQUAL(2u, schema->getNumAttributeFields()); - id = schema->getAttributeFieldId(name); - ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID); - EXPECT_EQUAL(timestamp, schema->getAttributeField(id).getTimestamp()); - - ASSERT_EQUAL(2u, schema->getNumSummaryFields()); - id = schema->getSummaryFieldId(name); - ASSERT_NOT_EQUAL(id, Schema::UNKNOWN_FIELD_ID); - EXPECT_EQUAL(timestamp, schema->getSummaryField(id).getTimestamp()); -} - -struct ListSchemaResult { - std::vector<vespalib::string> fieldNames; - std::vector<vespalib::string> fieldDataTypes; - std::vector<vespalib::string> fieldCollectionTypes; - std::vector<vespalib::string> fieldLocations; -}; - -void -assertSchemaResult(const vespalib::string &name, - const vespalib::string &dataType, - const vespalib::string &collectionType, - const vespalib::string &location, - const ListSchemaResult &r, - size_t i) -{ - EXPECT_EQUAL(name, r.fieldNames[i]); - EXPECT_EQUAL(dataType, r.fieldDataTypes[i]); - EXPECT_EQUAL(collectionType, r.fieldCollectionTypes[i]); - EXPECT_EQUAL(location, r.fieldLocations[i]); -} - -TEST("require that listSchema can list all fields") -{ - Schema schema; - schema.addIndexField(Schema::IndexField("if", DataType::STRING)); - schema.addAttributeField(Schema::AttributeField("af", DataType::INT32)); - schema.addSummaryField(Schema::SummaryField("sf", DataType::FLOAT, CollectionType::ARRAY)); - - ListSchemaResult r; - SchemaUtil::listSchema(schema, r.fieldNames, r.fieldDataTypes, r.fieldCollectionTypes, r.fieldLocations); - EXPECT_EQUAL(3u, r.fieldNames.size()); - EXPECT_EQUAL(3u, r.fieldDataTypes.size()); - EXPECT_EQUAL(3u, r.fieldCollectionTypes.size()); - EXPECT_EQUAL(3u, r.fieldLocations.size()); - assertSchemaResult("af", "INT32", "SINGLE", "a", r, 0); - assertSchemaResult("if", "STRING", "SINGLE", "i", r, 1); - assertSchemaResult("sf", "FLOAT", "ARRAY", "s", r, 2); -} - -} // namespace - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index e167979d065..2d6584bdc0c 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -1,12 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/searchcore/proton/common/selectpruner.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/cloningvisitor.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/test/mock_attribute_manager.h> @@ -60,7 +61,7 @@ const document::DocumentId docId("doc:test:1"); DocumentTypeRepo::UP -makeDocTypeRepo(void) +makeDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, @@ -93,7 +94,7 @@ rsString(const ResultSet &s) { std::ostringstream os; bool first = true; - uint32_t erange = Result::enumRange(); + uint32_t erange = Result::enumRange; for (uint32_t e = 0; e < erange; ++e) { if (s.hasEnum(e)) { if (!first) @@ -131,9 +132,9 @@ public: DocumentTypeRepo::UP _repoUP; bool _hasFields; - TestFixture(void); + TestFixture(); - ~TestFixture(void); + ~TestFixture(); void testParse(const string &selection); @@ -152,7 +153,7 @@ public: }; -TestFixture::TestFixture(void) +TestFixture::TestFixture() : _amgr(), _repoUP(), _hasFields(true) @@ -164,7 +165,7 @@ TestFixture::TestFixture(void) } -TestFixture::~TestFixture(void) +TestFixture::~TestFixture() { } diff --git a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp b/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp index 9b04a126a48..408c49ad0a1 100644 --- a/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp +++ b/searchcore/src/tests/proton/common/state_reporter_utils/state_reporter_utils_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("state_reporter_utils_test"); diff --git a/searchcore/src/tests/proton/create-test.sh b/searchcore/src/tests/proton/create-test.sh index 2e3b7fc4af9..4dc983794b5 100755..100644 --- a/searchcore/src/tests/proton/create-test.sh +++ b/searchcore/src/tests/proton/create-test.sh @@ -24,7 +24,6 @@ gen_source() { echo "generating '$1' ..." echo "#include <vespa/log/log.h>" > $1 echo "LOG_SETUP(\"${test}_test\");" >> $1 - echo "#include <vespa/fastos/fastos.h>" >> $1 echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1 echo "" >> $1 echo "// using namespace ;" >> $1 diff --git a/searchcore/src/tests/proton/docsummary/.gitignore b/searchcore/src/tests/proton/docsummary/.gitignore index f5e934f84da..80361b6c406 100644 --- a/searchcore/src/tests/proton/docsummary/.gitignore +++ b/searchcore/src/tests/proton/docsummary/.gitignore @@ -1,6 +1,5 @@ .depend Makefile docsummary_test - searchcore_docsummary_test_app searchcore_summaryfieldconverter_test_app diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index d7311a2cc63..6c36b004133 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -106,7 +106,7 @@ public: { } - ~BuildContext(void) + ~BuildContext() { } @@ -126,7 +126,7 @@ public: namespace { const char * -getDocTypeName(void) +getDocTypeName() { return "searchdocument"; } @@ -306,7 +306,7 @@ private: } const std::set<vespalib::string> & - getMarkupFields(void) const + getMarkupFields() const { return _markupFields; } @@ -952,13 +952,13 @@ Test::requireThatUrisAreUsed() startIndexField("urisingle"). startSubField("all"). addUrlTokenizedString( - "http://www.yahoo.com:81/fluke?ab=2#4"). + "http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("81"). @@ -977,13 +977,13 @@ Test::requireThatUrisAreUsed() startElement(1). startSubField("all"). addUrlTokenizedString( - "http://www.yahoo.com:82/fluke?ab=2#8"). + "http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("82"). @@ -1027,13 +1027,13 @@ Test::requireThatUrisAreUsed() startElement(4). startSubField("all"). addUrlTokenizedString( - "http://www.yahoo.com:83/fluke?ab=2#12"). + "http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("83"). @@ -1086,14 +1086,14 @@ Test::requireThatUrisAreUsed() bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"), getMarkupFields()); - EXPECT_TRUE(assertString("http://www.yahoo.com:81/fluke?ab=2#4", + EXPECT_TRUE(assertString("http://www.example.com:81/fluke?ab=2#4", "urisingle", dsa, 1)); GeneralResultPtr res = getResult(dsa, 1); { vespalib::Slime slime; decode(res->GetEntry("uriarray"), slime); EXPECT_TRUE(slime.get().valid()); - EXPECT_EQUAL("http://www.yahoo.com:82/fluke?ab=2#8", asVstring(slime.get()[0])); + EXPECT_EQUAL("http://www.example.com:82/fluke?ab=2#8", asVstring(slime.get()[0])); EXPECT_EQUAL("http://www.flickr.com:82/fluke?ab=2#9", asVstring(slime.get()[1])); } { @@ -1104,7 +1104,7 @@ Test::requireThatUrisAreUsed() EXPECT_EQUAL(7L, slime.get()[1]["weight"].asLong()); vespalib::string arr0s = asVstring(slime.get()[0]["item"]); vespalib::string arr1s = asVstring(slime.get()[1]["item"]); - EXPECT_EQUAL("http://www.yahoo.com:83/fluke?ab=2#12", arr0s); + EXPECT_EQUAL("http://www.example.com:83/fluke?ab=2#12", arr0s); EXPECT_EQUAL("http://www.flickr.com:85/fluke?ab=2#13", arr1s); } } diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp index 878833fa58b..b9b74527dca 100644 --- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp +++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp @@ -369,9 +369,9 @@ Document Test::makeDocument() { doc.setValue("position2", LongFieldValue(ZCurve::encode(4, 2))); StructFieldValue uri(getDataType("url")); - uri.setValue("all", annotateTerm("http://www.yahoo.com:42/foobar?q#frag")); + uri.setValue("all", annotateTerm("http://www.example.com:42/foobar?q#frag")); uri.setValue("scheme", annotateTerm("http")); - uri.setValue("host", annotateTerm("www.yahoo.com")); + uri.setValue("host", annotateTerm("www.example.com")); uri.setValue("port", annotateTerm("42")); uri.setValue("path", annotateTerm("foobar")); uri.setValue("query", annotateTerm("q")); @@ -379,10 +379,10 @@ Document Test::makeDocument() { doc.setValue("uri", uri); ArrayFieldValue uri_array(getDataType("Array<url>")); - uri.setValue("all", annotateTerm("http://www.yahoo.com:80/foobar?q#frag")); + uri.setValue("all", annotateTerm("http://www.example.com:80/foobar?q#frag")); uri.setValue("port", annotateTerm("80")); uri_array.add(uri); - uri.setValue("all", annotateTerm("https://www.yahoo.com:443/foo?q#frag")); + uri.setValue("all", annotateTerm("https://www.example.com:443/foo?q#frag")); uri.setValue("scheme", annotateTerm("https")); uri.setValue("path", annotateTerm("foo")); uri.setValue("port", annotateTerm("443")); diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index 0ccd5b8d6ee..8261ae80e02 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -2,6 +2,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/result.h> diff --git a/searchcore/src/tests/proton/documentdb/.gitignore b/searchcore/src/tests/proton/documentdb/.gitignore index abcba544a6d..a755b9966b4 100644 --- a/searchcore/src/tests/proton/documentdb/.gitignore +++ b/searchcore/src/tests/proton/documentdb/.gitignore @@ -2,5 +2,4 @@ Makefile .depend documentdb_test tmp - searchcore_documentdb_test_app diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp index 1910161cd32..c0ec7d93b9e 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/buckethandler/buckethandler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("buckethandler_test"); #include <vespa/searchcore/proton/server/buckethandler.h> diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp index c66e9f49274..58c1f773aea 100644 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/clusterstatehandler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchcore/proton/server/clusterstatehandler.h> #include <vespa/searchcore/proton/server/iclusterstatechangedhandler.h> #include <vespa/searchcore/proton/test/test.h> diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index 61796df8468..f04c2f08bb8 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("combiningfeedview_test"); #include <vespa/searchcore/proton/feedoperation/moveoperation.h> diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index 6ffe9b2494a..db95898e349 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/server/document_scan_iterator.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/documentid.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace document; using namespace proton; diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index abfd80c9582..633f7929a6d 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -16,14 +16,15 @@ #include <vespa/searchcore/proton/server/document_subdb_explorer.h> #include <vespa/searchcore/proton/server/emptysearchview.h> #include <vespa/searchcore/proton/server/fast_access_document_retriever.h> +#include <vespa/searchcore/proton/server/i_document_subdb_owner.h> #include <vespa/searchcore/proton/server/minimal_document_retriever.h> #include <vespa/searchcore/proton/server/searchabledocsubdb.h> -#include <vespa/searchcore/proton/server/i_document_subdb_owner.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> -#include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/common/lambdatask.h> +#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> @@ -40,10 +41,11 @@ using namespace search::transactionlog; using namespace cloud::config::filedistribution; using namespace searchcorespi; using namespace vespalib; + using proton::bucketdb::BucketDBHandler; using proton::bucketdb::IBucketDBHandler; using proton::bucketdb::IBucketDBHandlerInitializer; - +using search::test::DirectoryHandler; using searchcorespi::IFlushTarget; using searchcorespi::index::IThreadingService; using storage::spi::Timestamp; @@ -74,11 +76,6 @@ struct MySubDBOwner : public IDocumentSubDBOwner uint32_t _syncCnt; MySubDBOwner() : _syncCnt(0) {} void syncFeedView() override { ++_syncCnt; } - IIndexManagerFactory::SP - getIndexManagerFactory(const vespalib::stringref &name) const override { - (void) name; - return IIndexManagerFactory::SP(); - } vespalib::string getName() const override { return "owner"; } uint32_t getDistributionKey() const override { return -1; } }; @@ -295,7 +292,7 @@ struct FixtureBase typename Traits::Context _ctx; typename Traits::Schema _baseSchema; MyConfigSnapshot::UP _snapshot; - test::DirectoryHandler _baseDir; + DirectoryHandler _baseDir; typename Traits::SubDB _subDb; IFeedView::SP _tmpFeedView; FixtureBase() @@ -321,7 +318,7 @@ struct FixtureBase } template <typename FunctionType> void runInMaster(FunctionType func) { - test::runInMaster(_writeService, func); + proton::test::runInMaster(_writeService, func); } void init() { DocumentSubDbInitializer::SP task = @@ -685,23 +682,29 @@ assertTarget(const vespalib::string &name, TEST_F("require that flush targets can be retrieved", FastAccessFixture) { IFlushTarget::List targets = getFlushTargets(f); - EXPECT_EQUAL(4u, targets.size()); - EXPECT_EQUAL("subdb.attribute.attr1", targets[0]->getName()); - EXPECT_EQUAL("subdb.documentmetastore", targets[1]->getName()); - EXPECT_EQUAL("subdb.summary.compact", targets[2]->getName()); - EXPECT_EQUAL("subdb.summary.flush", targets[3]->getName()); + EXPECT_EQUAL(7u, targets.size()); + EXPECT_EQUAL("subdb.attribute.flush.attr1", targets[0]->getName()); + EXPECT_EQUAL("subdb.attribute.shrink.attr1", targets[1]->getName()); + EXPECT_EQUAL("subdb.documentmetastore.flush", targets[2]->getName()); + EXPECT_EQUAL("subdb.documentmetastore.shrink", targets[3]->getName()); + EXPECT_EQUAL("subdb.summary.compact", targets[4]->getName()); + EXPECT_EQUAL("subdb.summary.flush", targets[5]->getName()); + EXPECT_EQUAL("subdb.summary.shrink", targets[6]->getName()); } TEST_F("require that flush targets can be retrieved", SearchableFixture) { IFlushTarget::List targets = getFlushTargets(f); - EXPECT_EQUAL(6u, targets.size()); - EXPECT_TRUE(assertTarget("subdb.attribute.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0])); - EXPECT_TRUE(assertTarget("subdb.documentmetastore", FType::SYNC, FComponent::ATTRIBUTE, *targets[1])); - EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[2])); - EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[3])); - EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[4])); - EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[5])); + EXPECT_EQUAL(9u, targets.size()); + EXPECT_TRUE(assertTarget("subdb.attribute.flush.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0])); + EXPECT_TRUE(assertTarget("subdb.attribute.shrink.attr1", FType::GC, FComponent::ATTRIBUTE, *targets[1])); + EXPECT_TRUE(assertTarget("subdb.documentmetastore.flush", FType::SYNC, FComponent::ATTRIBUTE, *targets[2])); + EXPECT_TRUE(assertTarget("subdb.documentmetastore.shrink", FType::GC, FComponent::ATTRIBUTE, *targets[3])); + EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[4])); + EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[5])); + EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[6])); + EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[7])); + EXPECT_TRUE(assertTarget("subdb.summary.shrink", FType::GC, FComponent::DOCUMENT_STORE, *targets[8])); } TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture) @@ -737,7 +740,7 @@ struct DocumentHandler startAttributeField("attr2").addInt(attr2Value).endField().endDocument(); } PutOperation createPut(Document::UP doc, Timestamp timestamp, SerialNum serialNum) { - test::Document testDoc(Document::SP(doc.release()), 0, timestamp); + proton::test::Document testDoc(Document::SP(doc.release()), 0, timestamp); PutOperation op(testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDoc()); op.setSerialNum(serialNum); return op; @@ -747,7 +750,7 @@ struct DocumentHandler uint32_t targetSubDbId, SerialNum serialNum) { - test::Document testDoc(Document::SP(doc.release()), 0, timestamp); + proton::test::Document testDoc(Document::SP(doc.release()), 0, timestamp); MoveOperation op(testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDoc(), sourceDbdId, targetSubDbId); op.setSerialNum(serialNum); return op; @@ -989,4 +992,3 @@ TEST_MAIN() { TEST_RUN_ALL(); } - diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index c191b7dc4b5..d757e48e7d0 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -1,47 +1,47 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/testlib/receptor.h> -#include <vespa/searchcommon/common/schema.h> +#include <tests/proton/common/dummydbowner.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> #include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchcore/proton/docsummary/summaryflushtarget.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> -#include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchcore/proton/matching/querylimiter.h> +#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> +#include <vespa/searchcore/proton/metrics/metricswireservice.h> +#include <vespa/searchcore/proton/reference/document_db_reference_registry.h> +#include <vespa/searchcore/proton/reference/i_document_db_reference.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/document_db_explorer.h> #include <vespa/searchcore/proton/server/documentdb.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> -#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> -#include <vespa/searchcore/proton/metrics/metricswireservice.h> -#include <vespa/searchcore/proton/reference/document_db_reference_registry.h> -#include <vespa/searchcore/proton/reference/i_document_db_reference.h> #include <vespa/searchcorespi/index/indexflushtarget.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/translogserver.h> -#include <tests/proton/common/dummydbowner.h> -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/testlib/receptor.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/testkit/test_kit.h> using document::DocumentType; using document::DocumentTypeRepo; -using search::index::Schema; -using search::transactionlog::TransLogServer; +using document::DocumenttypesConfig; +using namespace cloud::config::filedistribution; using namespace proton; using namespace vespalib::slime; -using namespace cloud::config::filedistribution; using search::TuneFileDocumentDB; -using document::DocumenttypesConfig; using search::index::DummyFileHeaderContext; +using search::index::Schema; +using search::transactionlog::TransLogServer; +using searchcorespi::IFlushTarget; using searchcorespi::index::IndexFlushTarget; using vespa::config::search::core::ProtonConfig; using vespalib::Slime; -using searchcorespi::IFlushTarget; namespace { @@ -146,7 +146,7 @@ extractRealFlushTarget(const IFlushTarget *target) } TEST_F("requireThatIndexFlushTargetIsUsed", Fixture) { - std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets(); + auto targets = f._db->getFlushTargets(); ASSERT_TRUE(!targets.empty()); const IndexFlushTarget *index = 0; for (size_t i = 0; i < targets.size(); ++i) { @@ -176,7 +176,7 @@ size_t getNumTargets(const std::vector<IFlushTarget::SP> & targets) } TEST_F("requireThatFlushTargetsAreNamedBySubDocumentDB", Fixture) { - std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets(); + auto targets = f._db->getFlushTargets(); ASSERT_TRUE(!targets.empty()); for (const IFlushTarget::SP & target : f._db->getFlushTargets()) { vespalib::string name = target->getName(); @@ -187,7 +187,7 @@ TEST_F("requireThatFlushTargetsAreNamedBySubDocumentDB", Fixture) { } TEST_F("requireThatAttributeFlushTargetsAreUsed", Fixture) { - std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets(); + auto targets = f._db->getFlushTargets(); ASSERT_TRUE(!targets.empty()); size_t numAttrs = getNumTargets<FlushableAttribute>(targets); // attr1 defined in attributes.cfg @@ -195,21 +195,27 @@ TEST_F("requireThatAttributeFlushTargetsAreUsed", Fixture) { } TEST_F("requireThatDocumentMetaStoreFlushTargetIsUsed", Fixture) { - std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets(); + auto targets = f._db->getFlushTargets(); ASSERT_TRUE(!targets.empty()); - size_t numMetaStores = - getNumTargets<DocumentMetaStoreFlushTarget>(targets); - // document meta store + size_t numMetaStores = getNumTargets<DocumentMetaStoreFlushTarget>(targets); EXPECT_EQUAL(3u, numMetaStores); } TEST_F("requireThatSummaryFlushTargetsIsUsed", Fixture) { - std::vector<IFlushTarget::SP> targets = f._db->getFlushTargets(); + auto targets = f._db->getFlushTargets(); ASSERT_TRUE(!targets.empty()); size_t num = getNumTargets<SummaryFlushTarget>(targets); EXPECT_EQUAL(3u, num); } +TEST_F("require that shrink lid space flush targets are created", Fixture) { + auto targets = f._db->getFlushTargets(); + ASSERT_TRUE(!targets.empty()); + size_t num = getNumTargets<ShrinkLidSpaceFlushTarget>(targets); + // 1x attribute, 3x document meta store, 3x document store + EXPECT_EQUAL(1u + 3u + 3u, num); +} + TEST_F("requireThatCorrectStatusIsReported", Fixture) { StatusReport::UP report(f._db->reportStatus()); EXPECT_EQUAL("documentdb:typea", report->getComponent()); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index cc953423604..ef5a1d67cd7 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -18,7 +18,6 @@ using namespace vespa::config::search; using proton::matching::RankingConstants; using std::make_shared; using std::shared_ptr; -using vespa::config::search::core::RankingConstantsConfig; using document::config_builder::DocumenttypesConfigBuilderHelper; using document::config_builder::Struct; diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index 31d580b0756..24c5cbb5a11 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -13,7 +13,6 @@ using namespace search::index; using namespace vespa::config::search; using std::shared_ptr; using std::make_shared; -using vespa::config::search::core::RankingConstantsConfig; typedef shared_ptr<DocumentDBConfig> DDBCSP; diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 67eccb2fed2..351c4a3d405 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -111,7 +111,7 @@ struct MyOwner : public IFeedHandlerOwner virtual void onPerformPrune(SerialNum) override {} virtual bool - getAllowPrune(void) const override + getAllowPrune() const override { return _allowPrune; } @@ -394,7 +394,7 @@ struct MyFeedMetrics : public metrics::MetricSet { PerDocTypeFeedMetrics _feed; - MyFeedMetrics(void) + MyFeedMetrics() : metrics::MetricSet("myfeedmetrics", "", "My feed metrics", NULL), _feed(this) { diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index bdc0fc5f15c..280b617246b 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -690,6 +690,7 @@ struct FixtureBase void assertChangeNotified(document::GlobalId gid, uint32_t expLid) { _gidToLidChangeHandler->assertLid(gid, expLid); } + void populateBeforeCompactLidSpace(); }; @@ -720,6 +721,14 @@ FixtureBase::FixtureBase(TimeStamp visibilityDelay) FixtureBase::~FixtureBase() { _writeServiceReal.sync(); } + +void +FixtureBase::populateBeforeCompactLidSpace() +{ + putAndWait(makeDummyDocs(0, 2, 1000)); + removeAndWait(makeDummyDocs(1, 1, 2000)); +} + struct SearchableFeedViewFixture : public FixtureBase { SearchableFeedView fv; @@ -1135,17 +1144,16 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc "blocks lid space shrinkage until generation is no longer used", SearchableFeedViewFixture) { - EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver())); - CompactLidSpaceOperation op(0, 99); - op.setSerialNum(1); - f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); }); + f.populateBeforeCompactLidSpace(); + EXPECT_TRUE(assertThreadObserver(5, 3, f.writeServiceObserver())); + f.compactLidSpaceAndWait(2); // performIndexForceCommit in index thread, then completion callback // in master thread. - EXPECT_TRUE(assertThreadObserver(3, 1, f.writeServiceObserver())); - EXPECT_EQUAL(99u, f.metaStoreObserver()._compactLidSpaceLidLimit); - EXPECT_EQUAL(99u, f.getDocumentStore()._compactLidSpaceLidLimit); + EXPECT_TRUE(assertThreadObserver(7, 4, f.writeServiceObserver())); + EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit); + EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); - EXPECT_EQUAL(99u, f._docIdLimit.get()); + EXPECT_EQUAL(2u, f._docIdLimit.get()); } TEST_F("require that compactLidSpace() doesn't propagate to " @@ -1153,12 +1161,13 @@ TEST_F("require that compactLidSpace() doesn't propagate to " "blocks lid space shrinkage until generation is no longer used", SearchableFeedViewFixture) { - EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver())); - CompactLidSpaceOperation op(0, 99); + f.populateBeforeCompactLidSpace(); + EXPECT_TRUE(assertThreadObserver(5, 3, f.writeServiceObserver())); + CompactLidSpaceOperation op(0, 2); op.setSerialNum(0); f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); }); // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread - EXPECT_TRUE(assertThreadObserver(2, 0, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(6, 3, f.writeServiceObserver())); EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1167,8 +1176,9 @@ TEST_F("require that compactLidSpace() doesn't propagate to " TEST_F("require that compactLidSpace() propagates to attributeadapter", FastAccessFeedViewFixture) { - f.runInMaster([&] () { f.fv.handleCompactLidSpace(CompactLidSpaceOperation(0, 99)); }); - EXPECT_EQUAL(99u, f.maw._wantedLidLimit); + f.populateBeforeCompactLidSpace(); + f.compactLidSpaceAndWait(2); + EXPECT_EQUAL(2u, f.maw._wantedLidLimit); } diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 4fd2f7f0332..89d6716c081 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -8,7 +8,6 @@ #include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/searchcore/proton/common/schemautil.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/fileconfigmanager.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> @@ -63,7 +62,7 @@ saveBaseConfigSnapshot(const DocumentDBConfig &snap, SerialNum num) DocumentDBConfig::SP -makeEmptyConfigSnapshot(void) +makeEmptyConfigSnapshot() { return test::DocumentDBConfigBuilder(0, std::make_shared<Schema>(), "client", "test").build(); } diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp index 19f938188d3..81b5013c1fc 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("job_tracked_maintenance_test"); diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 6d97b7b0b1c..c300035cc4d 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("lid_space_compaction_test"); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp index 5dc72d02b15..737643c67ad 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/frozenbucketsmap_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("frozenbucketsmap_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index 7a28dd02bca..4dd6c454540 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for storeonlyfeedview. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/documentid.h> #include <vespa/document/base/globalid.h> diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index d247c313ad4..ef994989a7b 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> -#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchlib/attribute/attributefilesavetarget.h> #include <vespa/searchlib/fef/matchdatalayout.h> @@ -69,7 +69,7 @@ class ReverseGidCompare : public DocumentMetaStore::IGidCompare { GlobalId::BucketOrderCmp _comp; public: - ReverseGidCompare(void) + ReverseGidCompare() : IGidCompare(), _comp() { @@ -90,7 +90,7 @@ struct BoolVector : public std::vector<bool> { BoolVector &F() { push_back(false); return *this; } uint32_t - countTrue(void) const + countTrue() const { uint32_t res(0); for (uint32_t i = 0; i < size(); ++i) @@ -1874,13 +1874,10 @@ TEST("requireThatShrinkViaFlushTargetWorks") HwInfo hwInfo; vespalib::rmdir("dmsflush", true); vespalib::mkdir("dmsflush"); - IFlushTarget::SP ft(new DocumentMetaStoreFlushTarget(dms, - dummyTlsSyncer, - "dmsflush", - tuneFileAttributes, - fileHeaderContext, - hwInfo)); - + using Type = IFlushTarget::Type; + using Component = IFlushTarget::Component; + IFlushTarget::SP ft(std::make_shared<ShrinkLidSpaceFlushTarget> + ("documentmetastore.shrink", Type::GC, Component::ATTRIBUTE, 0, IFlushTarget::Time(), dms)); populate(10, *dms); uint32_t shrinkTarget = 5; diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 18c92312446..c35ac3c141a 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("lidreusedelayer_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 86bbf2ba94a..d1d966c515c 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("feed_and_search_test"); diff --git a/searchcore/src/tests/proton/feedoperation/.gitignore b/searchcore/src/tests/proton/feedoperation/.gitignore index 695cdf3365d..cfdeb9049b2 100644 --- a/searchcore/src/tests/proton/feedoperation/.gitignore +++ b/searchcore/src/tests/proton/feedoperation/.gitignore @@ -1,5 +1,4 @@ *_test .depend Makefile - searchcore_feedoperation_test_app diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 9c17b904eed..1c8165d9f17 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -25,6 +25,7 @@ #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/testkit/testapp.h> using document::BucketId; @@ -94,7 +95,7 @@ void assertDocumentOperation(DocumentOperation &op, BucketId expBucket, uint32_t } DocumentTypeRepo::UP -makeDocTypeRepo(void) +makeDocTypeRepo() { DocumenttypesConfigBuilderHelper builder; builder.document(doc_type_id, type_name, diff --git a/searchcore/src/tests/proton/flushengine/flushengine.cpp b/searchcore/src/tests/proton/flushengine/flushengine.cpp index 7de03c665a7..57302834441 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine.cpp @@ -231,7 +231,7 @@ public: } virtual search::SerialNum - getFlushSerial(void) const override + getFlushSerial() const override { return 0u; } @@ -421,7 +421,7 @@ public: _done.countDown(); } virtual search::SerialNum - getFlushSerial(void) const override + getFlushSerial() const override { return 0u; } diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index c91d1897984..b5e37240f62 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h> diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/CMakeLists.txt new file mode 100644 index 00000000000..8a917ba8e29 --- /dev/null +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_flushengine_shrink_lid_space_flush_target_test_app TEST + SOURCES + shrink_lid_space_flush_target_test.cpp + DEPENDS + searchcorespi + searchcore_flushengine +) +vespa_add_test( + NAME searchcore_flushengine_shrink_lid_space_flush_target_test_app + COMMAND searchcore_flushengine_shrink_lid_space_flush_target_test_app +) diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/DESC b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/DESC new file mode 100644 index 00000000000..e8cb3013c29 --- /dev/null +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/DESC @@ -0,0 +1 @@ +ShrinkLidSpaceFlushTarget test. Take a look at shrink_lid_space_flush_target_test.cpp for details. diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/FILES b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/FILES new file mode 100644 index 00000000000..54ff425acdd --- /dev/null +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/FILES @@ -0,0 +1 @@ +shrink_lid_space_flush_target_test.cpp diff --git a/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp new file mode 100644 index 00000000000..abf6b5e40df --- /dev/null +++ b/searchcore/src/tests/proton/flushengine/shrink_lid_space_flush_target/shrink_lid_space_flush_target_test.cpp @@ -0,0 +1,122 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> +#include <vespa/searchlib/common/i_compactable_lid_space.h> + +using namespace proton; +using search::SerialNum; +using searchcorespi::IFlushTarget; + +bool validTask(const std::unique_ptr<searchcorespi::FlushTask> &task) +{ + return static_cast<bool>(task); +} + +class MyLidSpace : public search::common::ICompactableLidSpace +{ + bool _canShrink; + size_t _canFree; +public: + MyLidSpace() + : _canShrink(true), + _canFree(16) + { + } + virtual ~MyLidSpace() override {} + + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { + (void) wantedDocLidLimit; + } + + virtual bool canShrinkLidSpace() const override { + return _canShrink; + } + + virtual size_t getEstimatedShrinkLidSpaceGain() const override { + return _canShrink ? _canFree : 0; + } + + virtual void shrinkLidSpace() override { + if (_canShrink) { + _canFree = 0; + _canShrink = false; + } + } + void setCanShrink(bool canShrink) { _canShrink = canShrink; } + void setCanFree(size_t canFree) { _canFree = canFree; } +}; + +struct Fixture +{ + std::shared_ptr<MyLidSpace> _lidSpace; + std::shared_ptr<ShrinkLidSpaceFlushTarget> _ft; + Fixture() + : _lidSpace(std::make_shared<MyLidSpace>()), + _ft(std::make_shared<ShrinkLidSpaceFlushTarget>("name", + IFlushTarget::Type::GC, + IFlushTarget::Component::ATTRIBUTE, + 10, + IFlushTarget::Time(), + _lidSpace)) + { + } + + ~Fixture() { } +}; + +TEST_F("require that flush target returns estimated memory gain", Fixture) +{ + auto memoryGain = f._ft->getApproxMemoryGain(); + EXPECT_EQUAL(16, memoryGain.gain()); + EXPECT_EQUAL(10u, f._ft->getFlushedSerialNum()); + EXPECT_EQUAL(IFlushTarget::Time(), f._ft->getLastFlushTime()); +} + +TEST_F("require that flush target returns no estimated memory gain when not able to flush", Fixture) +{ + f._lidSpace->setCanShrink(false); + auto memoryGain = f._ft->getApproxMemoryGain(); + EXPECT_EQUAL(0, memoryGain.gain()); +} + +TEST_F("require that flush target returns no estimated memory gain right after shrink", Fixture) +{ + auto task = f._ft->initFlush(20); + EXPECT_TRUE(validTask(task)); + task->run(); + auto memoryGain = f._ft->getApproxMemoryGain(); + EXPECT_EQUAL(0, memoryGain.gain()); + EXPECT_EQUAL(20u, f._ft->getFlushedSerialNum()); + EXPECT_NOT_EQUAL(IFlushTarget::Time(), f._ft->getLastFlushTime()); +} + +TEST_F("require that flush target returns no task when not able to flush", Fixture) +{ + f._lidSpace->setCanShrink(false); + auto task = f._ft->initFlush(20); + EXPECT_FALSE(validTask(task)); + EXPECT_EQUAL(20u, f._ft->getFlushedSerialNum()); + EXPECT_NOT_EQUAL(IFlushTarget::Time(), f._ft->getLastFlushTime()); +} + +TEST_F("require that flush target returns valid task when able to flush again", Fixture) +{ + f._lidSpace->setCanShrink(false); + auto task = f._ft->initFlush(20); + EXPECT_FALSE(validTask(task)); + EXPECT_EQUAL(20u, f._ft->getFlushedSerialNum()); + EXPECT_NOT_EQUAL(IFlushTarget::Time(), f._ft->getLastFlushTime()); + f._lidSpace->setCanShrink(true); + auto memoryGain = f._ft->getApproxMemoryGain(); + EXPECT_EQUAL(16, memoryGain.gain()); + task = f._ft->initFlush(20); + EXPECT_TRUE(validTask(task)); + task->run(); + EXPECT_EQUAL(20u, f._ft->getFlushedSerialNum()); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp index 594b718296c..225a977e384 100644 --- a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp +++ b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp @@ -1,16 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for diskindexcleaner. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("diskindexcleaner_test"); - #include <vespa/searchcorespi/index/activediskindexes.h> #include <vespa/searchcorespi/index/diskindexcleaner.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/fastos/file.h> #include <string> #include <vector> +#include <vespa/log/log.h> +LOG_SETUP("diskindexcleaner_test"); + using std::string; using std::vector; using namespace searchcorespi::index; diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index 1ff5a6e8649..9776923abb0 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("index_writer_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index cf26b05bf7d..fa7572d50bd 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for IndexManager. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("indexmanager_test"); - #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/searchcore/proton/index/indexmanager.h> @@ -29,8 +25,12 @@ LOG_SETUP("indexmanager_test"); #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/fastos/file.h> #include <set> +#include <vespa/log/log.h> +LOG_SETUP("indexmanager_test"); + using document::Document; using document::FieldValue; using search::SequencedTaskExecutor; diff --git a/searchcore/src/tests/proton/initializer/task_runner_test.cpp b/searchcore/src/tests/proton/initializer/task_runner_test.cpp index 8167d9f5242..064c481727a 100644 --- a/searchcore/src/tests/proton/initializer/task_runner_test.cpp +++ b/searchcore/src/tests/proton/initializer/task_runner_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("task_runner_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp index 7ddee5d4490..34a01256a42 100644 --- a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp +++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_bench.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/docid_range_scheduler.h> #include <vespa/vespalib/util/rendezvous.h> diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp index 7676d6f8598..35a51ee1aed 100644 --- a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp +++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/docid_range_scheduler.h> #include <chrono> diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index ff86e0a5ff1..bd9bb6707b6 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/value_cache/constant_value.h> diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp index 92139a1c027..65dbd291178 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/match_loop_communicator.h> #include <vespa/vespalib/util/box.h> diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp index b5f373efc9c..8c8e6d6026a 100644 --- a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp +++ b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/match_phase_limiter.h> #include <vespa/searchlib/queryeval/termasstring.h> diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 237f283f042..54c90557591 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("matching_stats_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp index 48b92c5ae46..61a73a464c3 100644 --- a/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp +++ b/searchcore/src/tests/proton/matching/partial_result/partial_result_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/partial_result.h> #include <vespa/vespalib/util/box.h> diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index d9c0fe09c24..bdb6a80830e 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querynodes. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("querynodes_test"); diff --git a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp index 2734e860c72..cbac81864dc 100644 --- a/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp +++ b/searchcore/src/tests/proton/matching/resolveviewvisitor_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for resolveviewvisitor. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("resolveviewvisitor_test"); diff --git a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp index 24767eb36f7..cb4bd7beee9 100644 --- a/searchcore/src/tests/proton/matching/sessionmanager_test.cpp +++ b/searchcore/src/tests/proton/matching/sessionmanager_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for sessionmanager. -#include <vespa/fastos/fastos.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/matching/session_manager_explorer.h> diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 43f0823d89f..1a65f7083e2 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for TermDataExtractor. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("termdataextractor_test"); diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp index 3269fe84dcd..816e214aa0c 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp +++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("documentdb_job_trackers_test"); #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("documentdb_job_trackers_test"); + using namespace proton; using namespace searchcorespi; @@ -22,6 +22,7 @@ struct MFT : public test::DummyFlushTarget }; struct AttributeFlush : public MFT { AttributeFlush() : MFT(FTT::SYNC, FTC::ATTRIBUTE) {} }; +struct AttributeShrink : public MFT { AttributeShrink() : MFT(FTT::GC, FTC::ATTRIBUTE) {} }; struct MemoryIndexFlush : public MFT { MemoryIndexFlush() : MFT(FTT::FLUSH, FTC::INDEX) {} }; struct DiskIndexFusion : public MFT { DiskIndexFusion() : MFT(FTT::GC, FTC::INDEX) {} }; struct DocStoreFlush : public MFT { DocStoreFlush() : MFT(FTT::SYNC, FTC::DOCUMENT_STORE) {} }; @@ -61,7 +62,7 @@ TEST_F("require that job metrics are updated", Fixture) // Update metrics 2 times to ensure that all jobs are running // in the last interval we actually care about. f._trackers.updateMetrics(f._metrics); - FastOS_Thread::Sleep(100); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); f._trackers.updateMetrics(f._metrics); EXPECT_APPROX(1.0, f._metrics.attributeFlush.getLast(), EPS); @@ -93,14 +94,16 @@ TEST_F("require that known flush targets are tracked", Fixture) input.push_back(IFlushTarget::SP(new DiskIndexFusion())); input.push_back(IFlushTarget::SP(new DocStoreFlush())); input.push_back(IFlushTarget::SP(new DocStoreCompaction())); + input.push_back(IFlushTarget::SP(new AttributeShrink())); IFlushTarget::List output = f._trackers.trackFlushTargets(input); - EXPECT_EQUAL(5u, output.size()); + EXPECT_EQUAL(6u, output.size()); EXPECT_TRUE(assertFlushTarget(f._trackers.getAttributeFlush(), *output[0])); EXPECT_TRUE(assertFlushTarget(f._trackers.getMemoryIndexFlush(), *output[1])); EXPECT_TRUE(assertFlushTarget(f._trackers.getDiskIndexFusion(), *output[2])); EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreFlush(), *output[3])); EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreCompact(), *output[4])); + EXPECT_TRUE(assertFlushTarget(f._trackers.getAttributeFlush(), *output[5])); } TEST_F("require that un-known flush targets are not tracked", Fixture) diff --git a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp b/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp index 21c5131b072..d122db9b82c 100644 --- a/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp +++ b/searchcore/src/tests/proton/metrics/job_load_sampler/job_load_sampler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("job_load_sampler_test"); diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp index edff0c4e2e4..4eee5e3e0b0 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp +++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("job_tracked_flush_test"); diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp index 78b53910023..4cabd619a3d 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp +++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for metrics_engine. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("metrics_engine_test"); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 07a86c0ccd2..8dea47e850f 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/persistence/spi/documentselection.h> #include <vespa/searchcore/proton/persistenceengine/bucket_guard.h> #include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h> diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 904cce00fc4..45c2ac10019 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -15,6 +15,7 @@ #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/i_proton_configurer.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/varholder.h> #include <mutex> diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 7a8247c8944..56c3e80ad17 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -16,6 +16,7 @@ #include <vespa/searchcore/proton/server/proton_configurer.h> #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/util/threadstackexecutor.h> diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 73dfae75d7b..5dfc0d41164 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -1,31 +1,33 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_reprocessing_initializer_test"); +#include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/attribute_populator.h> -#include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/attributedisklayout.h> -#include <vespa/searchcore/proton/attribute/attribute_directory.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/common/i_indexschema_inspector.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h> #include <vespa/searchcore/proton/test/attribute_utils.h> -#include <vespa/searchcore/proton/test/directory_handler.h> -#include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcore/proton/common/hw_info.h> -#include <vespa/searchlib/common/foregroundtaskexecutor.h> using namespace proton; using namespace search; using namespace search::index; -using search::attribute::Config; + +using proton::test::AttributeUtils; using search::attribute::BasicType; +using search::attribute::Config; using search::index::schema::DataType; +using search::test::DirectoryHandler; const vespalib::string TEST_DIR = "test_output"; const SerialNum INIT_SERIAL_NUM = 10; @@ -64,13 +66,13 @@ struct MyConfig void addAttrs(const StringVector &attrs) { for (auto attr : attrs) { if (attr == "tensor") { - _mgr->addAttribute({attr, test::AttributeUtils::getTensorConfig()}, 1); + _mgr->addAttribute({attr, AttributeUtils::getTensorConfig()}, 1); _schema.addAttributeField(Schema::AttributeField(attr, DataType::TENSOR)); } else if (attr == "predicate") { - _mgr->addAttribute({attr, test::AttributeUtils::getPredicateConfig()}, 1); + _mgr->addAttribute({attr, AttributeUtils::getPredicateConfig()}, 1); _schema.addAttributeField(Schema::AttributeField(attr, DataType::BOOLEANTREE)); } else { - _mgr->addAttribute({attr, test::AttributeUtils::getStringConfig()}, 1); + _mgr->addAttribute({attr, AttributeUtils::getStringConfig()}, 1); _schema.addAttributeField(Schema::AttributeField(attr, DataType::STRING)); } } @@ -125,7 +127,7 @@ struct MyIndexschemaInspector : public IIndexschemaInspector struct Fixture { - test::DirectoryHandler _dirHandler; + DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp index fb1434d4a0c..06419bd9404 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("document_reprocessing_handler_test"); diff --git a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp index 4d9cbd307ba..0d239cd2472 100644 --- a/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/reprocessing_runner/reprocessing_runner_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("reprocessing_runner_test"); @@ -54,7 +53,7 @@ struct MyTask : public IReprocessingTask } virtual Progress - getProgress(void) const override + getProgress() const override { return Progress(_myProgress, _weight); } diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore b/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore deleted file mode 100644 index b085eedc970..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_data_directory_upgrader_test_app diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt b/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt deleted file mode 100644 index ffcc7a82d2e..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_data_directory_upgrader_test_app TEST - SOURCES - data_directory_upgrader_test.cpp - DEPENDS - searchcore_server - searchcore_flushengine -) -vespa_add_test(NAME searchcore_data_directory_upgrader_test_app COMMAND searchcore_data_directory_upgrader_test_app) diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/DESC b/searchcore/src/tests/proton/server/data_directory_upgrader/DESC deleted file mode 100644 index d0ca4c99210..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/DESC +++ /dev/null @@ -1 +0,0 @@ -data_directory_upgrader test. Take a look at data_directory_upgrader_test.cpp for details. diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/FILES b/searchcore/src/tests/proton/server/data_directory_upgrader/FILES deleted file mode 100644 index d1aee9bddfa..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/FILES +++ /dev/null @@ -1 +0,0 @@ -data_directory_upgrader_test.cpp diff --git a/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp b/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp deleted file mode 100644 index 7b6cf9143ee..00000000000 --- a/searchcore/src/tests/proton/server/data_directory_upgrader/data_directory_upgrader_test.cpp +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("data_directory_upgrader_test"); -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/searchcore/proton/server/data_directory_upgrader.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <iostream> - -using namespace proton; -using namespace vespalib; - -typedef DataDirectoryUpgrader::RowColDir RowColDir; -typedef DataDirectoryUpgrader::ScanResult ScanResult; -typedef DataDirectoryUpgrader::UpgradeResult UpgradeResult; - -const string SCAN_DIR = "mytest"; -const string DEST_DIR = SCAN_DIR + "/n1"; - -void -assertDirs(const DirectoryList &exp, const DirectoryList &act) -{ - ASSERT_EQUAL(exp.size(), act.size()); - for (size_t i = 0; i < exp.size(); ++i) { - EXPECT_EQUAL(exp[i], act[i]); - } -} - -void -assertDirs(const DirectoryList &rowColDirs, bool destDirExisting, const ScanResult &act) -{ - ASSERT_EQUAL(rowColDirs.size(), act.getRowColDirs().size()); - for (size_t i = 0; i < rowColDirs.size(); ++i) { - EXPECT_EQUAL(rowColDirs[i], act.getRowColDirs()[i].dir()); - } - EXPECT_EQUAL(destDirExisting, act.isDestDirExisting()); -} - -void -assertDataFile(const vespalib::string &dir) -{ - FileInfo::UP file = stat(dir + "/data.txt"); - ASSERT_TRUE(file.get() != NULL); - EXPECT_TRUE(file->_plainfile); -} - -vespalib::string -readFile(const vespalib::string &fileName) -{ - File file(fileName); - file.open(File::READONLY); - FileInfo info = file.stat(); - char buf[512]; - size_t bytesRead = file.read(&buf, info._size, 0); - return vespalib::string(buf, bytesRead); -} - -void -assertUpgradeFile(const vespalib::string &exp, const vespalib::string &dir) -{ - EXPECT_EQUAL(exp, readFile(dir + "/data-directory-upgrade-source.txt")); -} - -void -assertDowngradeScript(const vespalib::string &exp, const vespalib::string &dir) -{ - EXPECT_EQUAL(exp, readFile(dir + "/data-directory-downgrade.sh")); -} - -struct BaseFixture -{ - DataDirectoryUpgrader _upg; - BaseFixture(const DirectoryList &dirs, bool createDestDir = false) : _upg(SCAN_DIR, DEST_DIR) { - mkdir(SCAN_DIR); - if (createDestDir) { - mkdir(DEST_DIR); - } - for (const string &dir : dirs) { - mkdir(SCAN_DIR + "/" + dir); - File f(SCAN_DIR + "/" + dir + "/data.txt"); - f.open(File::CREATE); - f.close(); - } - } - virtual ~BaseFixture() { - rmdir(SCAN_DIR, true); - } - DirectoryList getDirs(const vespalib::string &subDir = "") const { - DirectoryList l = listDirectory(SCAN_DIR + "/" + subDir); - std::sort(l.begin(), l.end()); - return l; - } -}; - -struct EmptyFixture : public BaseFixture -{ - EmptyFixture() : BaseFixture({}) {} -}; - -struct SingleFixture : public BaseFixture -{ - SingleFixture() : BaseFixture({"r0/c0"}) {} -}; - -struct DoubleFixture : public BaseFixture -{ - DoubleFixture() : BaseFixture({"r0/c0", "r1/c1"}) {} -}; - -struct UnrelatedFixture : public BaseFixture -{ - UnrelatedFixture() : BaseFixture({"r0/cY", "rX/c1", "r0"}) {} -}; - -struct ExistingDestinationFixture : public BaseFixture -{ - ExistingDestinationFixture() : BaseFixture({"r0/c0"}, true) {} -}; - -TEST_F("require that single row/column directory is discovered", SingleFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0"}, false, res); -} - -TEST_F("require that multiple row/column directories are discovered", DoubleFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0", "r1/c1"}, false, res); -} - -TEST_F("require that unrelated directories are not discovered", UnrelatedFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({}, false, res); -} - -TEST_F("require that existing destination directory is discovered", ExistingDestinationFixture) -{ - ScanResult res = f._upg.scan(); - assertDirs({"r0/c0"}, true, res); -} - -TEST("require that no-existing scan directory is handled") -{ - DataDirectoryUpgrader upg(SCAN_DIR, DEST_DIR); - ScanResult res = upg.scan(); - assertDirs({}, false, res); -} - -TEST_F("require that empty directory is left untouched", EmptyFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus()); - EXPECT_EQUAL("No directory to upgrade", res.getDesc()); - DirectoryList dirs = f.getDirs(); - assertDirs({}, dirs); -} - -TEST_F("require that existing destination directory is left untouched", ExistingDestinationFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::IGNORE, res.getStatus()); - EXPECT_EQUAL("Destination directory 'mytest/n1' is already existing", res.getDesc()); - DirectoryList dirs = f.getDirs(); - assertDirs({"n1", "r0"}, dirs); -} - -TEST_F("require that single directory is upgraded", SingleFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::COMPLETE, res.getStatus()); - EXPECT_EQUAL("Moved data from 'mytest/r0/c0' to 'mytest/n1'", res.getDesc()); - DirectoryList dirs = f.getDirs(); - std::sort(dirs.begin(), dirs.end()); - assertDirs({"n1"}, dirs); - assertDataFile(DEST_DIR); - assertUpgradeFile("mytest/r0/c0", DEST_DIR); - assertDowngradeScript("#!/bin/sh\n\n" - "mkdir mytest/r0 || exit 1\n" - "chown yahoo mytest/r0\n" - "mv mytest/n1 mytest/r0/c0\n" - "rm mytest/r0/c0/data-directory-upgrade-source.txt\n" - "rm mytest/r0/c0/data-directory-downgrade.sh\n", DEST_DIR); -} - -TEST_F("require that multiple directories are left untouched", DoubleFixture) -{ - UpgradeResult res = f._upg.upgrade(f._upg.scan()); - EXPECT_EQUAL(DataDirectoryUpgrader::ERROR, res.getStatus()); - EXPECT_EQUAL("Can only upgrade a single directory, was asked to upgrade 2 ('r0/c0', 'r1/c1')", res.getDesc()); - DirectoryList dirs = f.getDirs(); - std::sort(dirs.begin(), dirs.end()); - assertDirs({"r0", "r1"}, dirs); - assertDataFile(SCAN_DIR + "/r0/c0"); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index e6c573cabe0..b0541f9ac71 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for documentretriever. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/documentid.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> @@ -30,11 +30,7 @@ #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/docstore/cachestats.h> -#include <vespa/searchlib/docstore/idocumentstore.h> -#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> using document::ArrayFieldValue; using document::FieldValue; diff --git a/searchcore/src/tests/proton/server/feeddebugger_test.cpp b/searchcore/src/tests/proton/server/feeddebugger_test.cpp index dfb1e7aa5ef..1cc70755ad2 100644 --- a/searchcore/src/tests/proton/server/feeddebugger_test.cpp +++ b/searchcore/src/tests/proton/server/feeddebugger_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("feeddebugger_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/base/documentid.h> #include <vespa/searchcore/proton/common/feeddebugger.h> diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index baf5819fdaf..c0e02efa90d 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("feedstates_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/base/documentid.h> #include <vespa/document/base/testdocrepo.h> diff --git a/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp b/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp index 866ba2c77c0..b77c6e24ba7 100644 --- a/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp +++ b/searchcore/src/tests/proton/server/health_adapter/health_adapter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/server/health_adapter.h> #include <vespa/searchcore/proton/common/statusreport.h> diff --git a/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp b/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp index a696855dd72..5a178f5a3e2 100644 --- a/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp +++ b/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_metrics_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index 8fc1bd2d9ed..94471176194 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -3,13 +3,11 @@ #include <vespa/log/log.h> LOG_SETUP("memoryconfigstore_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcore/proton/common/schemautil.h> using search::SerialNum; using search::index::Schema; diff --git a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp index 2b6b5bccee7..c940909b569 100644 --- a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp +++ b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("visibility_handler_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index dae8cce36e4..80b78d420ec 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h> @@ -11,6 +10,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/document/util/compressor.h> #include <vespa/searchlib/common/transport.h> +#include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> LOG_SETUP("summaryengine_test"); diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 2c5e119ecbe..8439c2fb54f 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -9,7 +9,7 @@ #include <map> #include <initializer_list> -const char *prog = "../../../apps/verify_ranksetup/verify_ranksetup-bin"; +const char *prog = "../../../apps/verify_ranksetup/vespa-verify-ranksetup-bin"; const std::string gen_dir("generated"); const char *valid_feature = "value(0)"; diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 81c0d0bc747..c851d124970 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -307,9 +307,8 @@ filedistributor.configid reference default="" restart ## Interval between pruning of old removed documents. ## -## Default value is 6 hours (21600 seconds). -## If set to 0, the value is calculated as (pruneremoveddocumentsage / 100) (default 3.36 hours). -pruneremoveddocumentsinterval double default=21600.0 +## If set to 0 (default) the value is calculated as (pruneremoveddocumentsage / 100) (default 3.36 hours). +pruneremoveddocumentsinterval double default=0.0 ## Age of removed document before it can be pruned. ## diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp index 15655d38cce..fbb71f81348 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp @@ -1,13 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".appcontext"); #include "appcontext.h" - FastS_TimeKeeper::FastS_TimeKeeper() : _clock(0.010), _thread_pool(128 * 1024) diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h index 948572bf616..1768ab8bc71 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h @@ -10,7 +10,7 @@ public: /** * Destructor. No cleanup needed for base class. */ - virtual ~FastS_IProperties(void) { } + virtual ~FastS_IProperties() { } virtual bool IsSet (const char *key) = 0; virtual bool BoolVal (const char *key) = 0; diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp index 34f7b1a3a55..55f3b6d895b 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2005 Overture Services Norway AS -#include <vespa/fastos/fastos.h> +#include "queryperf.h" + #include <vespa/log/log.h> LOG_SETUP(".queryperf"); -#include <vespa/fnet/fnet.h> -#include <vespa/searchcore/fdispatch/common/queryperf.h> - FastS_QueryPerf::FastS_QueryPerf() : queueLen(0), diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h index 36d52cec820..a2b5593d9d8 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/queryperf.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2005 Overture Services Norway AS #pragma once +#include <cstdint> + struct FastS_QueryPerf { uint32_t queueLen; diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp index 94d881ad075..39b2c74a4a9 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp @@ -1,16 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".rpc"); -#include <vespa/fnet/frt/frt.h> - -#include <vespa/searchcore/util/log.h> #include "rpc.h" -#include <vespa/searchsummary/docsummary/resultconfig.h> +#include "appcontext.h" +#include <vespa/fnet/info.h> extern char FastS_VersionTag[]; @@ -22,6 +15,14 @@ FastS_RPC::FastS_RPC(FastS_AppContext *appCtx) : { } +bool FastS_RPC::Start() { + return _transport.Start(_appCtx->GetThreadPool()); +} + +void FastS_RPC::ShutDown() { + _transport.ShutDown(true); +} + bool FastS_RPC::Init(int port, const vespalib::string &myHeartbeatId) { diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h index 9693d31314e..2a374d7d0d7 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h @@ -1,14 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once -#include <vespa/fnet/frt/frt.h> -#include <vespa/searchcore/fdispatch/common/appcontext.h> +#include <vespa/fnet/frt/invokable.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/transport.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/slobrok/sbregister.h> +class FastS_AppContext; + class FastS_RPC : public FRT_Invokable { private: @@ -22,13 +23,13 @@ private: public: FastS_RPC(FastS_AppContext *appCtx); - virtual ~FastS_RPC() {} + ~FastS_RPC() {} FastS_AppContext *GetAppCtx() { return _appCtx; } FRT_Supervisor *GetSupervisor() { return &_supervisor; } bool Init(int port, const vespalib::string& myHeartbeatId); - bool Start() { return _transport.Start(_appCtx->GetThreadPool()); } - void ShutDown() { _transport.ShutDown(true); } + bool Start(); + void ShutDown(); // Register RPC Methods diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp index e4c713bf901..79097cc7326 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp @@ -1,11 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".search"); -#include <vespa/searchcore/util/log.h> #include "search.h" //--------------------------------------------------------------------- @@ -161,7 +155,6 @@ FastS_SyncSearchAdapter::Adapt(FastS_ISearch *search) return search; FastS_SyncSearchAdapter *ret = new FastS_SyncSearchAdapter(search); - FastS_assert(ret != NULL); search->SetAsyncArgs(ret, FastS_SearchContext()); return ret; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.h b/searchcore/src/vespa/searchcore/fdispatch/common/search.h index 52c30b7a9d7..a018187dfdb 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/search.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.h @@ -1,16 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once -#include <vespa/document/base/globalid.h> +#include <vespa/searchsummary/docsummary/getdocsumargs.h> #include <vespa/searchlib/common/fslimits.h> #include <vespa/searchlib/engine/errorcodes.h> -#include <vespa/searchsummary/docsummary/getdocsumargs.h> #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/common/packets.h> - +#include <vespa/document/base/globalid.h> +#include <vespa/fastos/cond.h> #include <limits> class FastS_ISearch; @@ -50,7 +48,7 @@ public: /** * Destructor. No cleanup needed for base class. */ - virtual ~FastS_ISearchOwner(void) { } + virtual ~FastS_ISearchOwner() { } virtual void DoneQuery(FastS_ISearch *search, FastS_SearchContext context) = 0; @@ -177,7 +175,7 @@ public: /** * Destructor. No cleanup needed for base class. */ - virtual ~FastS_ISearch(void) { } + virtual ~FastS_ISearch() { } enum RetCode { diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp index 57f951dc482..7f76e4e4777 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/searchcore/fdispatch/common/timestat.h> +#include "timestat.h" void FastS_TimeStatHistory::Reset() @@ -20,7 +16,7 @@ FastS_TimeStatHistory::Reset() double -FastS_TimeStatHistory::GetMaxTime(void) const +FastS_TimeStatHistory::GetMaxTime() const { double max = 0.0; uint32_t idx = _sampleIdx; diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h index ab22fd18e61..ae707ea9700 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h @@ -12,7 +12,7 @@ public: uint32_t _totalTimeouts; double _totalAccTime; - FastS_TimeStatTotals(void) + FastS_TimeStatTotals() : _totalCount(0), _totalTimeouts(0u), _totalAccTime(0) @@ -38,7 +38,7 @@ public: } double - getAvgTime(void) const + getAvgTime() const { if (_totalCount == 0) return 0.0; @@ -46,7 +46,7 @@ public: } double - getTimeoutRate(void) const + getTimeoutRate() const { if (_totalCount == 0) return 0.0; @@ -153,7 +153,7 @@ class FastS_TimeStatHistory return static_cast<double>(timeIdx) * SLOT_SIZE; } public: - FastS_TimeStatHistory(void) + FastS_TimeStatHistory() : _sampleAccTime(0.0), _totalAccTime(0.0), _sampleIdx(0), @@ -169,13 +169,13 @@ public: Reset(); } - void Reset(void); + void Reset(); - double GetSampleAccTime(void) const { return _sampleAccTime; } - uint32_t GetSampleCount(void) const { return _sampleCount; } + double GetSampleAccTime() const { return _sampleAccTime; } + uint32_t GetSampleCount() const { return _sampleCount; } uint32_t - getSampleTimeouts(void) const + getSampleTimeouts() const { return _sampleTimeouts; } @@ -186,15 +186,15 @@ public: return 0.0; return (_sampleAccTime / _sampleCount); } - double GetMaxTime(void) const; + double GetMaxTime() const; void Update(double tnow, double t, bool timedout); - uint32_t GetTotalCount(void) const { return _totalCount; } - double GetTotalAccTime(void) const { return _totalAccTime; } + uint32_t GetTotalCount() const { return _totalCount; } + double GetTotalAccTime() const { return _totalAccTime; } uint32_t - getTotalTimeouts(void) const + getTotalTimeouts() const { return _totalTimeouts; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp index 32362dba422..771a0a675c0 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fdispatch.docsumadapter"); #include "docsumadapter.h" -#include <vespa/searchcore/util/log.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/fef/queryproperties.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/searchcore/fdispatch/search/datasetcollection.h> +#include <vespa/log/log.h> +LOG_SETUP(".fdispatch.docsumadapter"); + namespace fdispatch { void @@ -34,9 +30,9 @@ void DocsumAdapter::handleRequest() { _dsc = _appCtx->GetDataSetCollection(); - FastS_assert(_dsc != NULL); + assert(_dsc != NULL); _search = _dsc->CreateSearch(FastS_NoID32(), _appCtx->GetTimeKeeper()); - FastS_assert(_search != NULL); + assert(_search != NULL); _docsumsResult = _search->GetDocsumsResult(); _search->SetGetDocsumArgs(&_args); _search->GetDocsums(_hitbuf, _hitcnt); diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp index 5c24b693ad8..86b4ddd00a1 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp @@ -1,17 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fdispatch.engineadapter"); #include "engineadapter.h" -#include <vespa/searchlib/common/packets.h> +#include "searchadapter.h" +#include "docsumadapter.h" #include <vespa/searchcore/fdispatch/search/child_info.h> #include <vespa/searchcore/fdispatch/search/nodemanager.h> -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> #include <vespa/searchcore/fdispatch/search/dataset_base.h> -#include <vespa/searchcore/fdispatch/common/search.h> -#include "searchadapter.h" -#include "docsumadapter.h" + +#include <vespa/log/log.h> +LOG_SETUP(".fdispatch.engineadapter"); namespace fdispatch { diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index 3be1834a360..c4a222af3e1 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -77,7 +77,7 @@ FastS_FNETAdapter::fini() } } -Fdispatch::~Fdispatch(void) +Fdispatch::~Fdispatch() { if (_transportServer) { _transportServer->shutDown(); // sync shutdown @@ -135,13 +135,13 @@ Fdispatch::GetThreadPool() } bool -Fdispatch::Failed(void) +Fdispatch::Failed() { return ( (_transportServer && _transportServer->isFailed())) || _needRestart; } bool -Fdispatch::CheckTempFail(void) +Fdispatch::CheckTempFail() { bool ret; bool failflag = _nodeManager->GetTempFail(); @@ -281,7 +281,7 @@ convert(InternalFdispatchrcType::Packetcompresstype type) } bool -Fdispatch::Init(void) +Fdispatch::Init() { int maxthreads; diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h index 00c10d5cfa9..58f68dfa9af 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h @@ -98,14 +98,14 @@ public: virtual FastOS_ThreadPool *GetThreadPool() override; virtual void logPerformance() override; virtual uint32_t getDispatchLevel() override; - bool CheckTempFail(void); - bool Failed(void); - bool Init(void); + bool CheckTempFail(); + bool Failed(); + bool Init(); int getHealthPort() const { return _healthPort; } vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; } Fdispatch(const config::ConfigUri &configUri); - ~Fdispatch(void); + ~Fdispatch(); }; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp index 5f571f0ecd1..d82f68fcd7d 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp @@ -1,26 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <functional> - -#include <vespa/log/log.h> -LOG_SETUP(".rpc"); - -#include <vespa/fnet/frt/frt.h> - -#include <vespa/searchcore/util/log.h> -#include <vespa/searchlib/common/transport.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> -#include <vespa/searchcore/fdispatch/search/configdesc.h> +#include "rpc.h" #include <vespa/searchcore/fdispatch/search/engine_base.h> -#include <vespa/searchcore/fdispatch/search/plain_dataset.h> #include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/program/rpc.h> - - void FastS_fdispatch_RPC::RegisterMethods(FRT_ReflectionBuilder *rb) { diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h index dad78a00b13..fc01d88615c 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once -#include <vespa/fnet/frt/frt.h> #include <vespa/searchcore/fdispatch/common/rpc.h> diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h index 9f2a65c2f91..9103ef2e06f 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h @@ -38,17 +38,17 @@ public: void SetNext(FastS_EngineDesc *next) { _next = next; } void SetConfPartID(int32_t value) { assert(value >= 0); _confPartID = value; } - void SetConfPartIDOverrides(void) { _confPartIDOverrides = true; } + void SetConfPartIDOverrides() { _confPartIDOverrides = true; } void SetConfRowID(int32_t value) { assert(value >= 0); _confRowID = value; } void SetUnitRefCost(uint32_t value) { _unitrefcost = value; } - void MarkBad(void) { _isBad = true; } - FastS_EngineDesc * GetNext(void) const { return _next; } - const char * GetName(void) const { return _name.c_str(); } - uint32_t GetConfPartID(void) const { return _confPartID; } - bool GetConfPartIDOverrides(void) const { return _confPartIDOverrides; } - uint32_t GetConfRowID(void) const { return _confRowID; } - uint32_t GetUnitRefCost(void) const { return _unitrefcost; } - bool IsBad(void) const { return _isBad; } + void MarkBad() { _isBad = true; } + FastS_EngineDesc * GetNext() const { return _next; } + const char * GetName() const { return _name.c_str(); } + uint32_t GetConfPartID() const { return _confPartID; } + bool GetConfPartIDOverrides() const { return _confPartIDOverrides; } + uint32_t GetConfRowID() const { return _confRowID; } + uint32_t GetUnitRefCost() const { return _unitrefcost; } + bool IsBad() const { return _isBad; } }; //----------------------------------------------------------------------- @@ -168,9 +168,9 @@ private: uint32_t _mpp; // Minimum number of engines per partition public: explicit FastS_DataSetDesc(uint32_t datasetid); - ~FastS_DataSetDesc(void); + ~FastS_DataSetDesc(); - uint32_t GetID(void) const { return _id; } + uint32_t GetID() const { return _id; } void SetUnitRefCost(uint32_t value) { _unitRefCost = value; } void SetPartBits(uint32_t value) { @@ -219,40 +219,40 @@ public: QueryDistributionMode GetQueryDistributionMode() { return _queryDistributionMode; } FastS_EngineDesc * AddEngine(const char *name); - uint32_t GetUnitRefCost(void) const { return _unitRefCost; } - uint32_t GetPartBits(void) const { return _partBits; } - - uint32_t GetRowBits(void) const { return _rowBits; } - uint32_t GetNumParts(void) const { return _numParts; } - uint32_t GetFirstPart(void) const { return _firstPart; } - uint32_t GetMinChildParts(void) const { return _minChildParts; } - uint32_t getMaxNodesDownPerFixedRow(void) const { return _maxNodesDownPerFixedRow; } - bool useRoundRobinForFixedRow(void) const { return _useRoundRobinForFixedRow; } - uint32_t GetMaxHitsPerNode(void) const { return _maxHitsPerNode; } - uint32_t GetEstimateParts(void) const { return _estimateParts; } - uint32_t GetEstPartCutoff(void) const { return _estPartCutoff; } - bool IsEstimatePartsSet(void) const { return _estimatePartsSet; } - bool IsEstPartCutoffSet(void) const { return _estPartCutoffSet; } - uint32_t GetMinOurActive(void) const { return _minOurActive; } - uint32_t GetMaxOurActive(void) const { return _maxOurActive; } - uint32_t GetCutoffOurActive(void) const { return _cutoffOurActive; } - uint32_t GetMinEstActive(void) const { return _minEstActive; } - uint32_t GetMaxEstActive(void) const { return _maxEstActive; } - uint32_t GetCutoffEstActive(void) const { return _cutoffEstActive; } - double GetQueueDrainRate(void) const { return _queueDrainRate; } - double GetQueueMaxDrain(void) const { return _queueMaxDrain; } - double GetSlowQueryLimitFactor(void) const { return _slowQueryLimitFactor; } - double GetSlowQueryLimitBias(void) const { return _slowQueryLimitBias; } - double GetSlowDocsumLimitFactor(void) const { return _slowDocsumLimitFactor; } - double GetSlowDocsumLimitBias(void) const { return _slowDocsumLimitBias; } - uint32_t GetEngineCnt(void) const { return _engineCnt; } - FastS_EngineDesc * GetEngineList(void) const { return _enginesHead; } + uint32_t GetUnitRefCost() const { return _unitRefCost; } + uint32_t GetPartBits() const { return _partBits; } + + uint32_t GetRowBits() const { return _rowBits; } + uint32_t GetNumParts() const { return _numParts; } + uint32_t GetFirstPart() const { return _firstPart; } + uint32_t GetMinChildParts() const { return _minChildParts; } + uint32_t getMaxNodesDownPerFixedRow() const { return _maxNodesDownPerFixedRow; } + bool useRoundRobinForFixedRow() const { return _useRoundRobinForFixedRow; } + uint32_t GetMaxHitsPerNode() const { return _maxHitsPerNode; } + uint32_t GetEstimateParts() const { return _estimateParts; } + uint32_t GetEstPartCutoff() const { return _estPartCutoff; } + bool IsEstimatePartsSet() const { return _estimatePartsSet; } + bool IsEstPartCutoffSet() const { return _estPartCutoffSet; } + uint32_t GetMinOurActive() const { return _minOurActive; } + uint32_t GetMaxOurActive() const { return _maxOurActive; } + uint32_t GetCutoffOurActive() const { return _cutoffOurActive; } + uint32_t GetMinEstActive() const { return _minEstActive; } + uint32_t GetMaxEstActive() const { return _maxEstActive; } + uint32_t GetCutoffEstActive() const { return _cutoffEstActive; } + double GetQueueDrainRate() const { return _queueDrainRate; } + double GetQueueMaxDrain() const { return _queueMaxDrain; } + double GetSlowQueryLimitFactor() const { return _slowQueryLimitFactor; } + double GetSlowQueryLimitBias() const { return _slowQueryLimitBias; } + double GetSlowDocsumLimitFactor() const { return _slowDocsumLimitFactor; } + double GetSlowDocsumLimitBias() const { return _slowDocsumLimitBias; } + uint32_t GetEngineCnt() const { return _engineCnt; } + FastS_EngineDesc * GetEngineList() const { return _enginesHead; } void setMPP(uint32_t mpp) { _mpp = mpp; } - uint32_t getMPP(void) const { return _mpp; } + uint32_t getMPP() const { return _mpp; } void setMonitorInterval(double monitorInterval) { _monitorInterval = monitorInterval; } - double getMonitorInterval(void) const { return _monitorInterval; } + double getMonitorInterval() const { return _monitorInterval; } void setHigherCoverageMaxSearchWait(double higherCoverageMaxSearchWait) { @@ -260,7 +260,7 @@ public: } double - getHigherCoverageMaxSearchWait(void) const { + getHigherCoverageMaxSearchWait() const { return _higherCoverageMaxSearchWait; } @@ -270,7 +270,7 @@ public: } double - getHigherCoverageMinSearchWait(void) const { + getHigherCoverageMinSearchWait() const { return _higherCoverageMinSearchWait; } @@ -280,7 +280,7 @@ public: } double - getHigherCoverageBaseSearchWait(void) const { + getHigherCoverageBaseSearchWait() const { return _higherCoverageBaseSearchWait; } @@ -290,7 +290,7 @@ public: } double - getMinimalSearchCoverage(void) const { + getMinimalSearchCoverage() const { return _minimalSearchCoverage; } @@ -300,7 +300,7 @@ public: } double - getHigherCoverageMaxDocSumWait(void) const { + getHigherCoverageMaxDocSumWait() const { return _higherCoverageMaxDocSumWait; } @@ -310,7 +310,7 @@ public: } double - getHigherCoverageMinDocSumWait(void) const { + getHigherCoverageMinDocSumWait() const { return _higherCoverageMinDocSumWait; } @@ -320,7 +320,7 @@ public: } double - getHigherCoverageBaseDocSumWait(void) const { + getHigherCoverageBaseDocSumWait() const { return _higherCoverageBaseDocSumWait; } @@ -330,11 +330,11 @@ public: } double - getMinimalDocSumCoverage(void) const { + getMinimalDocSumCoverage() const { return _minimalDocSumCoverage; } - void FinalizeConfig(void); + void FinalizeConfig(); }; //----------------------------------------------------------------------- @@ -351,18 +351,18 @@ private: bool _frozen; bool _error; - void HandleDeprecatedFPEstPartsOption(void); - bool CheckIntegrity(void); + void HandleDeprecatedFPEstPartsOption(); + bool CheckIntegrity(); public: - FastS_DataSetCollDesc(void); - ~FastS_DataSetCollDesc(void); + FastS_DataSetCollDesc(); + ~FastS_DataSetCollDesc(); FastS_DataSetDesc *LookupCreateDataSet(uint32_t datasetid); - bool Freeze(void); + bool Freeze(); - uint32_t GetMaxNumDataSets(void) const { return _datasets_size; } + uint32_t GetMaxNumDataSets() const { return _datasets_size; } FastS_DataSetDesc *GetDataSet(uint32_t datasetid) const { return (datasetid < _datasets_size) diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp index 34c8e75c5c0..6645cf493ec 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp @@ -68,7 +68,7 @@ FastS_DataSetBase::queryQueue_t::QueueTail(queryQueued_t *newqueued) void -FastS_DataSetBase::queryQueue_t::DeQueueHead(void) +FastS_DataSetBase::queryQueue_t::DeQueueHead() { queryQueued_t *queued = _head; FastS_assert(_queueLen > 0); @@ -112,7 +112,7 @@ FastS_DataSetBase::ScheduleCheckTempFail() void -FastS_DataSetBase::DeQueueHeadWakeup_HasLock(void) +FastS_DataSetBase::DeQueueHeadWakeup_HasLock() { queryQueued_t *queued; queued = _queryQueue.GetFirst(); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h index 13201210b79..92dda228487 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h @@ -101,28 +101,28 @@ public: { } - ~queryQueued_t(void) + ~queryQueued_t() { FastS_assert(!_isQueued); } - void Wait(void) { + void Wait() { _queueCond.Lock(); while (_isQueued) { _queueCond.Wait(); } _queueCond.Unlock(); } - bool IsAborted(void) const { return _isAborted; } - void MarkAbort(void) { _isAborted = true; } - void MarkQueued(void) { _isQueued = true; } - void UnmarkQueued(void) { _isQueued = false; } - bool IsQueued(void) const { return _isQueued; } - void LockCond(void) { _queueCond.Lock(); } - void UnlockCond(void) { _queueCond.Unlock(); } - void SignalCond(void) { _queueCond.Signal(); } + bool IsAborted() const { return _isAborted; } + void MarkAbort() { _isAborted = true; } + void MarkQueued() { _isQueued = true; } + void UnmarkQueued() { _isQueued = false; } + bool IsQueued() const { return _isQueued; } + void LockCond() { _queueCond.Lock(); } + void UnlockCond() { _queueCond.Unlock(); } + void SignalCond() { _queueCond.Signal(); } FNET_Task * - getDequeuedTask(void) const + getDequeuedTask() const { return _deQueuedTask; } @@ -154,12 +154,12 @@ public: queryQueue_t(FastS_DataSetDesc *desc); ~queryQueue_t(); void QueueTail(queryQueued_t *newquery); - void DeQueueHead(void); + void DeQueueHead(); unsigned int GetQueueLen() const { return _queueLen; } unsigned int GetActiveQueries() const { return _active; } void SetActiveQuery() { _active++; } void ClearActiveQuery() { _active--; } - queryQueued_t *GetFirst(void) const { return _head; } + queryQueued_t *GetFirst() const { return _head; } }; //---------------------------------------------------------------- @@ -209,9 +209,9 @@ public: void CountTimeout(); void ScheduleCheckTempFail(); - virtual void DeQueueHeadWakeup_HasLock(void); + virtual void DeQueueHeadWakeup_HasLock(); virtual ChildInfo getChildInfo() const; - uint32_t GetMldDocStamp(void) const { return _mldDocStamp; } + uint32_t GetMldDocStamp() const { return _mldDocStamp; } void SetMldDocStamp(uint32_t mldDocStamp) { _mldDocStamp = mldDocStamp; } // common dataset API diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp index af296cadf1c..cab6b68d5f4 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp @@ -256,7 +256,7 @@ FastS_DataSetCollection::CheckQueryQueues(FastS_TimeKeeper *timeKeeper) void -FastS_DataSetCollection::AbortQueryQueues(void) +FastS_DataSetCollection::AbortQueryQueues() { for (uint32_t datasetidx(0); datasetidx < GetMaxNumDataSets(); datasetidx++) { FastS_DataSetBase *dataset = PeekDataSet(datasetidx); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h index 1c1db158d27..3381c380ed1 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h @@ -82,8 +82,8 @@ public: FastS_ISearch *CreateSearch(uint32_t dataSetID, FastS_TimeKeeper *timeKeeper); // handle old query queues - bool IsLastRef(void) { return (refCount() == 1); } + bool IsLastRef() { return (refCount() == 1); } void CheckQueryQueues(FastS_TimeKeeper *timeKeeper); - void AbortQueryQueues(void); + void AbortQueryQueues(); }; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp index 5ca7bcd9103..44dcb02106b 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp @@ -53,7 +53,7 @@ FastS_EngineBase::reported_t::reported_t() } -FastS_EngineBase::reported_t::~reported_t(void) +FastS_EngineBase::reported_t::~reported_t() { } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h index d65367115d0..912c6c7594a 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h @@ -161,7 +161,7 @@ public: bool IsUp() const { return _isUp; } bool IsRealBad() const { return (_badness > BAD_NOT); } - bool isAdminBad(void) const { return _badness == BAD_ADMIN; } + bool isAdminBad() const { return _badness == BAD_ADMIN; } bool IsReady() const { return (IsUp() || IsRealBad()); } void SlowQuery(double limit, double secs, bool silent); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp index 9f775fcce51..ebd077ece46 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp @@ -1,22 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> +#include "fnet_dataset.h" +#include "fnet_engine.h" +#include "fnet_search.h" +#include "datasetcollection.h" + #include <vespa/log/log.h> LOG_SETUP(".search.fnet_dataset"); -#include <vespa/fnet/fnet.h> -#include <vespa/searchlib/common/fslimits.h> -#include <vespa/searchcore/util/log.h> -#include <vespa/searchcore/fdispatch/search/configdesc.h> -#include <vespa/searchcore/fdispatch/common/search.h> -#include <vespa/searchcore/util/eventloop.h> - -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/search/fnet_dataset.h> -#include <vespa/searchcore/fdispatch/search/fnet_engine.h> -#include <vespa/searchcore/fdispatch/search/fnet_search.h> //-------------------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h index 7f5fbd50298..48ea6b58b2f 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h @@ -4,8 +4,9 @@ #pragma once -#include <vespa/searchcore/fdispatch/search/plain_dataset.h> +#include "plain_dataset.h" +class FNET_Transport; class FastS_FNET_DataSet : public FastS_PlainDataSet { diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp index bd4c585f27e..c7b2ee6d5fa 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp @@ -1,28 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".search.fnet_engine"); -#include <vespa/fnet/fnet.h> -#include <vespa/searchcore/fdispatch/common/stdincl.h> -#include <vespa/searchlib/common/fslimits.h> -#include <vespa/searchcore/util/log.h> -#include <vespa/searchcore/fdispatch/search/configdesc.h> -#include <vespa/searchcore/fdispatch/common/search.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchcore/fdispatch/search/fnet_dataset.h> -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/search/fnet_engine.h> +#include "fnet_dataset.h" +#include "datasetcollection.h" +#include "fnet_engine.h" +#include <vespa/searchcore/fdispatch/common/search.h> +#include <vespa/fnet/transport.h> +#include <vespa/fnet/connection.h> using namespace search::fs4transport; //---------------------------------------------------------------------- void -FastS_StaticMonitorQuery::Free(void) +FastS_StaticMonitorQuery::Free() { _lock.Lock(); _refcnt--; @@ -34,16 +24,14 @@ FastS_StaticMonitorQuery::Free(void) } -FastS_StaticMonitorQuery::FastS_StaticMonitorQuery(void) +FastS_StaticMonitorQuery::FastS_StaticMonitorQuery() : FS4Packet_MONITORQUERYX(), _lock(), _refcnt(1) -{ - -} +{ } -FastS_StaticMonitorQuery::~FastS_StaticMonitorQuery(void) +FastS_StaticMonitorQuery::~FastS_StaticMonitorQuery() { FastS_assert(_refcnt == 0); } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h index 068c072e844..d5fa8167a98 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h @@ -18,19 +18,19 @@ class FastS_StaticMonitorQuery : public FS4Packet_MONITORQUERYX public: virtual void Free() override; - bool getBusy(void) const + bool getBusy() const { return _refcnt > 1; } - void markBusy(void) + void markBusy() { _lock.Lock(); _refcnt++; _lock.Unlock(); } - FastS_StaticMonitorQuery(void); - ~FastS_StaticMonitorQuery(void); + FastS_StaticMonitorQuery(); + ~FastS_StaticMonitorQuery(); }; //---------------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 592c4f053e4..16a123eb8e7 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -584,7 +584,7 @@ FastS_FNET_Search::~FastS_FNET_Search() void -FastS_FNET_Search::dropDatasetActiveCostRef(void) +FastS_FNET_Search::dropDatasetActiveCostRef() { if (_datasetActiveCostRef) { _dataset->SubCost(); @@ -1483,7 +1483,7 @@ FastS_FNET_Search::adjustQueryTimeout() void -FastS_FNET_Search::adjustDocsumTimeout(void) +FastS_FNET_Search::adjustDocsumTimeout() { uint32_t pendingDocsums = getPendingDocsums(); diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h index af747d887e0..1b8ed0dcb2c 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h @@ -103,15 +103,15 @@ public: uint32_t NT_GetNumHits() const { return (_hit_end - _hit_beg); } void NT_NextHit() { _hit_cur++; } - uint32_t getPartID(void) const { return _partid; } - uint32_t GetRowID(void) const { return _rowid; } - uint32_t GetTimeStamp(void) const { return _stamp; } + uint32_t getPartID() const { return _partid; } + uint32_t GetRowID() const { return _rowid; } + uint32_t GetTimeStamp() const { return _stamp; } FastS_FNET_SearchNode * allocExtraDocsumNode(bool mld, uint32_t rowid, uint32_t rowbits); - FastS_FNET_Engine *GetEngine(void) const { return _engine; } + FastS_FNET_Engine *GetEngine() const { return _engine; } - bool IsConnected(void) const { return _channel != NULL; } + bool IsConnected() const { return _channel != NULL; } void Connect(FastS_FNET_Engine *engine); void Connect_HasDSLock(FastS_FNET_Engine *engine); FastS_EngineBase * getPartition(const FastOS_Mutex & dsMutex, bool userow, FastS_FNET_DataSet *dataset); @@ -126,7 +126,7 @@ public: return (_engine == NULL ? 0 : _engine->getPortNumber()); } - void dropCost(void) { + void dropCost() { if (_engine != NULL && _flags._needSubCost) { _engine->SubCost(); _flags._needSubCost = false; @@ -174,13 +174,13 @@ public: { } - ExtraDocsumNodesIter & operator++(void) { + ExtraDocsumNodesIter & operator++() { _cur = _cur->_nextExtraDocsumNode; return *this; } - bool valid(void) const { return _cur != _head; } - FastS_FNET_SearchNode *operator*(void) const { return _cur; } + bool valid() const { return _cur != _head; } + FastS_FNET_SearchNode *operator*() const { return _cur; } }; @@ -258,7 +258,7 @@ private: std::vector<FastS_fullresult> _resbuf; - void dropDatasetActiveCostRef(void); + void dropDatasetActiveCostRef(); typedef std::vector<std::pair<FastS_EngineBase *, FastS_FNET_SearchNode *>> EngineNodeMap; void connectNodes(const EngineNodeMap & engines); @@ -284,7 +284,7 @@ private: FS4Packet_QUERYRESULTX::FS4_hit *pt, FS4Packet_QUERYRESULTX::FS4_hit *end); - FastS_TimeKeeper *GetTimeKeeper(void) const { return _timeKeeper; } + FastS_TimeKeeper *GetTimeKeeper() const { return _timeKeeper; } FastS_FNET_SearchNode * getNode(size_t i) { return &_nodes[i]; } public: @@ -345,23 +345,23 @@ public: FastS_QueryResult *ST_GetQueryResult() { return _util.GetQueryResult(); } void adjustQueryTimeout(); - void adjustDocsumTimeout(void); - uint32_t getGoodQueries(void) const { return _goodQueries; } - uint32_t getRequestedQueries(void) const { return _queryNodes; } - uint32_t getPendingQueries(void) const { return _pendingQueries; } - uint32_t getDoneQueries(void) const { + void adjustDocsumTimeout(); + uint32_t getGoodQueries() const { return _goodQueries; } + uint32_t getRequestedQueries() const { return _queryNodes; } + uint32_t getPendingQueries() const { return _pendingQueries; } + uint32_t getDoneQueries() const { return getRequestedQueries() - getPendingQueries(); } - uint32_t getBadQueries(void) const { + uint32_t getBadQueries() const { return getDoneQueries() - getGoodQueries(); } - uint32_t getGoodDocsums(void) const { return _goodDocsums; } - uint32_t getRequestedDocsums(void) const { return _requestedDocsums; } - uint32_t getPendingDocsums(void) const { return _pendingDocsums; } - uint32_t getDoneDocsums(void) const { + uint32_t getGoodDocsums() const { return _goodDocsums; } + uint32_t getRequestedDocsums() const { return _requestedDocsums; } + uint32_t getPendingDocsums() const { return _pendingDocsums; } + uint32_t getDoneDocsums() const { return getRequestedDocsums() - getPendingDocsums(); } - uint32_t getBadDocsums(void) const { + uint32_t getBadDocsums() const { return getDoneDocsums() - getGoodDocsums(); } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp index 9639cc5a8bd..a3a9afd8dcd 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp @@ -1,20 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(""); -#include <vespa/fnet/fnet.h> -#include <vespa/searchcore/util/log.h> +#include "mergehits.h" +#include "querycacheutil.h" +#include "fnet_dataset.h" +#include "fnet_search.h" #include <vespa/searchcore/util/stlishheap.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchcore/fdispatch/common/appcontext.h> -#include <vespa/searchcore/fdispatch/search/querycacheutil.h> -#include <vespa/searchcore/fdispatch/search/fnet_dataset.h> -#include <vespa/searchcore/fdispatch/search/fnet_engine.h> -#include <vespa/searchcore/fdispatch/search/fnet_search.h> -#include <vespa/searchcore/fdispatch/search/mergehits.h> + +#include <vespa/log/log.h> +LOG_SETUP(".fdispatch.mergehits"); using search::common::SortData; using search::common::SortDataIterator; @@ -212,8 +205,7 @@ FastS_HitMerger<T>::MergeHits() if (sortDataDocs == numDocs) { useSortData = true; } else { - LOG(warning, - "Some results are missing sort data, sorting by rank instead"); + LOG(warning, "Some results are missing sort data, sorting by rank instead"); } } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h index cbdda7e39bf..75a72c84948 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h @@ -1,13 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once +#include <vespa/searchlib/common/sortdata.h> +#include <vespa/searchlib/common/packets.h> #include <vespa/document/base/globalid.h> //----------------------------------------------------------------------------- +class FastS_hitresult; +class FastS_QueryResult; +class FastS_FNET_Search; +class FastS_FNET_SearchNode; + // T::HitType API struct FastS_MergeHits_DummyHit @@ -29,10 +34,8 @@ struct FastS_MergeHits_DummyHit struct FastS_MergeHits_DummyNode { - bool NT_InitMerge(uint32_t *numDocs, - uint64_t *totalHits, - search::HitRank *maxRank, - uint32_t *sortDataDocs) + bool NT_InitMerge(uint32_t *numDocs, uint64_t *totalHits, + search::HitRank *maxRank, uint32_t *sortDataDocs) { (void) numDocs; (void) totalHits; @@ -40,7 +43,7 @@ struct FastS_MergeHits_DummyNode (void) sortDataDocs; return false; } - search::common::SortDataIterator *NT_GetSortDataIterator() { return NULL; } + search::common::SortDataIterator *NT_GetSortDataIterator() { return NULL; } FastS_MergeHits_DummyHit *NT_GetHit() { return NULL; } uint32_t NT_GetNumHitsUsed() { return 0; } uint32_t NT_GetNumHitsLeft() { return 0; } @@ -85,7 +88,7 @@ struct FastS_MergeHits_DummyMerge struct FastS_FNETMerge { - typedef FS4Packet_QUERYRESULTX::FS4_hit HitType; + typedef search::fs4transport::FS4Packet_QUERYRESULTX::FS4_hit HitType; typedef FastS_FNET_SearchNode NodeType; typedef FastS_FNET_Search SearchType; }; @@ -100,7 +103,6 @@ private: FastS_HitMerger& operator=(const FastS_HitMerger &); - typedef typename T::HitType HIT; typedef typename T::NodeType NODE; typedef typename T::SearchType SEARCH; @@ -153,6 +155,3 @@ public: void MergeHits(); }; - -//----------------------------------------------------------------------------- - diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp index 9435df7fa62..ed5e069c3ee 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp @@ -1,17 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> - -#include <vespa/searchcore/fdispatch/common/appcontext.h> -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/search/nodemanager.h> -#include <vespa/searchcore/fdispatch/search/plain_dataset.h> +#include "nodemanager.h" +#include "datasetcollection.h" +#include "plain_dataset.h" #include "engine_base.h" +#include <vespa/searchcore/fdispatch/common/appcontext.h> +#include <vespa/log/log.h> LOG_SETUP(".search.nodemanager"); void diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h index 8543456b4f6..1d2aa617f35 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h @@ -1,6 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once @@ -13,6 +11,9 @@ using vespa::config::search::core::PartitionsConfig; class FastS_DataSetBase; +class FastS_AppContext; +class FastS_DataSetCollection; +class FastS_TimeKeeper; class FastS_NodeManager : public config::IFetcherCallback<PartitionsConfig> { @@ -22,9 +23,9 @@ private: vespalib::SimpleComponentConfigProducer &_componentConfig; - FastOS_Mutex _managerLock; - FastOS_Mutex _configLock; - FastOS_Mutex _stampLock; + FastOS_Mutex _managerLock; + FastOS_Mutex _configLock; + FastOS_Mutex _stampLock; FastS_AppContext *_appCtx; uint32_t _mldPartit; uint32_t _mldDocStamp; // Bumped for all cache flushes diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp index af2d6627ac9..11fc2cc3f8a 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp @@ -1,26 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <functional> +#include "plain_dataset.h" +#include "datasetcollection.h" +#include "engine_base.h" +#include "nodemanager.h" +#include <vespa/searchcore/fdispatch/common/search.h> +#include <vespa/vespalib/util/host_name.h> #include <iomanip> -#include <sstream> #include <vespa/log/log.h> -#include <vespa/fnet/fnet.h> -#include <vespa/vespalib/util/host_name.h> -#include <vespa/searchcore/fdispatch/common/stdincl.h> -#include <vespa/searchlib/common/fslimits.h> -#include <vespa/searchcore/util/log.h> -#include <vespa/searchcore/fdispatch/search/configdesc.h> -#include <vespa/searchcore/fdispatch/common/search.h> - -#include <vespa/searchcore/fdispatch/search/datasetcollection.h> -#include <vespa/searchcore/fdispatch/search/engine_base.h> -#include <vespa/searchcore/fdispatch/search/plain_dataset.h> -#include <vespa/searchcore/fdispatch/search/nodemanager.h> - LOG_SETUP(".search.plain_dataset"); //-------------------------------------------------------------------------- @@ -70,9 +58,7 @@ FastS_PartitionMap::FastS_PartitionMap(FastS_DataSetDesc *desc) { // finalize config settings if (_num_partitions > (1U << _partBits)) { - LOG(error, - "Too many partitions %d constrained by partbits %d", - _num_partitions, _partBits); + LOG(error, "Too many partitions %d constrained by partbits %d", _num_partitions, _partBits); _num_partitions = (1U << _partBits); } @@ -567,19 +553,9 @@ FastS_PlainDataSet::AreEnginesReady() void FastS_PlainDataSet::Ping() { -#ifdef FNET_SANITY_CHECKS - FastOS_Time beforePing; - beforePing.SetNow(); -#endif for (FastS_EngineBase* engine : _enginesArray) { engine->Ping(); } -#ifdef FNET_SANITY_CHECKS - double pingTime = beforePing.MilliSecsToNow(); - if (pingTime > 100.0) { - LOG(warning, "SANITY: Dataset (%d) ping time: %.2f ms", GetID(), pingTime); - } -#endif } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h index 62aa7b33166..0aab44f0625 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h @@ -150,21 +150,21 @@ public: } uint32_t getNumRows() const { return _partMap.getNumRows(); } uint32_t getNumPartitions(size_t rowId) { return _partMap.getNumPartitions(rowId); } - uint32_t GetRowBits(void) const { return _partMap._rowBits; } - uint32_t GetPartBits(void) const { return _partMap._partBits; } - uint32_t GetFirstPart(void) const { return _partMap._first_partition; } - uint32_t GetLastPart(void) const { + uint32_t GetRowBits() const { return _partMap._rowBits; } + uint32_t GetPartBits() const { return _partMap._partBits; } + uint32_t GetFirstPart() const { return _partMap._first_partition; } + uint32_t GetLastPart() const { return _partMap._first_partition + _partMap._num_partitions; } - uint32_t GetPartitions(void) const { return _partMap._num_partitions; } - uint32_t GetEstimateParts(void) const { return _estimateParts; } - uint32_t GetEstimatePartCutoff(void) const { return _estimatePartCutoff; } - uint32_t GetMaxHitsPerNode(void) const { return _maxHitsPerNode; } - double GetSlowQueryLimitFactor(void) const { return _slowQueryLimitFactor; } - double GetSlowQueryLimitBias(void) const { return _slowQueryLimitBias; } - double GetSlowDocsumLimitFactor(void) const { return _slowDocsumLimitFactor; } - double GetSlowDocsumLimitBias(void) const { return _slowDocsumLimitBias; } - bool GetTempFail(void) const { return _partMap._childparts < _partMap._minchildparts; } + uint32_t GetPartitions() const { return _partMap._num_partitions; } + uint32_t GetEstimateParts() const { return _estimateParts; } + uint32_t GetEstimatePartCutoff() const { return _estimatePartCutoff; } + uint32_t GetMaxHitsPerNode() const { return _maxHitsPerNode; } + double GetSlowQueryLimitFactor() const { return _slowQueryLimitFactor; } + double GetSlowQueryLimitBias() const { return _slowQueryLimitBias; } + double GetSlowDocsumLimitFactor() const { return _slowDocsumLimitFactor; } + double GetSlowDocsumLimitBias() const { return _slowDocsumLimitBias; } + bool GetTempFail() const { return _partMap._childparts < _partMap._minchildparts; } void UpdateMaxHitsPerNodeLog(bool incomplete, bool fuzzy); uint32_t getMaxNodesDownPerFixedRow() const { return _partMap._maxNodesDownPerFixedRow; } uint32_t useRoundRobinForFixedRow() const { return _partMap._useRoundRobinForFixedRow; } @@ -191,16 +191,16 @@ public: virtual ChildInfo getChildInfo() const override; - uint32_t getMPP(void) const { return _partMap._mpp; } - double getMonitorInterval(void) const { return _monitorInterval; } - double getHigherCoverageMaxSearchWait(void) const { return _higherCoverageMaxSearchWait; } - double getHigherCoverageMinSearchWait(void) const { return _higherCoverageMinSearchWait; } - double getHigherCoverageBaseSearchWait(void) const { return _higherCoverageBaseSearchWait; } - double getMinimalSearchCoverage(void) const { return _minimalSearchCoverage; } - double getHigherCoverageMaxDocSumWait(void) const { return _higherCoverageMaxDocSumWait; } - double getHigherCoverageMinDocSumWait(void) const { return _higherCoverageMinDocSumWait; } - double getHigherCoverageBaseDocSumWait(void) const { return _higherCoverageBaseDocSumWait; } - double getMinimalDocSumCoverage(void) const { return _minimalDocSumCoverage; } + uint32_t getMPP() const { return _partMap._mpp; } + double getMonitorInterval() const { return _monitorInterval; } + double getHigherCoverageMaxSearchWait() const { return _higherCoverageMaxSearchWait; } + double getHigherCoverageMinSearchWait() const { return _higherCoverageMinSearchWait; } + double getHigherCoverageBaseSearchWait() const { return _higherCoverageBaseSearchWait; } + double getMinimalSearchCoverage() const { return _minimalSearchCoverage; } + double getHigherCoverageMaxDocSumWait() const { return _higherCoverageMaxDocSumWait; } + double getHigherCoverageMinDocSumWait() const { return _higherCoverageMinDocSumWait; } + double getHigherCoverageBaseDocSumWait() const { return _higherCoverageBaseDocSumWait; } + double getMinimalDocSumCoverage() const { return _minimalDocSumCoverage; } // API //---- @@ -210,7 +210,7 @@ public: // Downcast //--------- - virtual FastS_PlainDataSet * GetPlainDataSet(void) override { return this; } + virtual FastS_PlainDataSet * GetPlainDataSet() override { return this; } template <class FUN> FUN ForEachEngine(FUN fun) { diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp index 73057061391..c59164d06f4 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp @@ -1,19 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".query"); -#include <vespa/searchcore/util/log.h> -#include <vespa/searchcore/fdispatch/search/query.h> -#include <vespa/searchlib/common/transport.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> +#include "query.h" +#include <vespa/searchlib/parsequery/simplequerystack.h> /** Marks as empty */ -FastS_query::FastS_query(void) +FastS_query::FastS_query() : _dataset(0), _flags(0), _stackDump(), @@ -57,7 +49,7 @@ FastS_query::getPrintableQuery() return _printableQuery.c_str(); } -FastS_query::~FastS_query(void) +FastS_query::~FastS_query() { } @@ -95,7 +87,7 @@ class SizeCollector : public search::fef::IPropertiesVisitor static const size_t _mapFuzz = 15; size_t _size; public: - SizeCollector(void) + SizeCollector() : _size(0) { } @@ -121,7 +113,7 @@ public: } size_t - getSize(void) const + getSize() const { return _size; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h index aeea70125b0..077b5b1b226 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/query.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.h @@ -23,9 +23,9 @@ public: FastS_query(const FastS_query &other); FastS_query &operator=(const FastS_query &other); public: - FastS_query(void); + FastS_query(); FastS_query(const search::docsummary::GetDocsumArgs &docsumArgs); - ~FastS_query(void); + ~FastS_query(); void SetStackDump(const vespalib::stringref& stackDump); void SetSortSpec(const char *spec) { _sortSpec = spec; } @@ -42,7 +42,7 @@ public: const search::fef::Properties &GetRankProperties() const { return _rankProperties; } const search::fef::Properties &GetFeatureOverrides() const { return _featureOverrides; } - uint32_t GetQueryFlags(void) const { return _flags; } + uint32_t GetQueryFlags() const { return _flags; } const char *getPrintableQuery(); bool IsFlagSet(uint32_t flag) const { return (_flags & flag) != 0; } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp index 7e95cf91325..ce0158897da 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp @@ -1,18 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".search.querycacheutil"); -#include <vespa/searchcore/util/log.h> - -#include <vespa/searchlib/common/transport.h> +#include "querycacheutil.h" #include <vespa/searchlib/parsequery/simplequerystack.h> - #include <vespa/searchlib/common/sortdata.h> -#include <vespa/searchcore/fdispatch/search/querycacheutil.h> +#include <vespa/log/log.h> +LOG_SETUP(".search.querycacheutil"); using search::common::SortData; @@ -115,7 +108,7 @@ FastS_QueryCacheUtil::AdjustSearchParametersFinal(uint32_t partitions) } void -FastS_QueryCacheUtil::DropResult(void) +FastS_QueryCacheUtil::DropResult() { _queryResult._groupResultLen = 0; _queryResult._groupResult = NULL; @@ -148,13 +141,13 @@ FastS_QueryCacheUtil::DropResult(void) bool -FastS_QueryCacheUtil::IsEstimate(void) const +FastS_QueryCacheUtil::IsEstimate() const { return _query.IsFlagSet(search::fs4transport::QFLAG_ESTIMATE); } void -FastS_QueryCacheUtil::InitEstimateMode(void) +FastS_QueryCacheUtil::InitEstimateMode() { _searchInfo._searchOffset = 0; _searchInfo._maxHits = 0; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h index 0ef3fb0abf7..0b3373efcd0 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h @@ -47,35 +47,35 @@ public: public: FastS_QueryCacheUtil(); ~FastS_QueryCacheUtil(); - bool AgeDropCheck(void); - void DropResult(void); - bool GotNoResultsYet(void) const { return _queryResult._hitbuf == NULL; } - uint32_t GetSearchOffset(void) const { return _searchInfo._searchOffset; } - uint32_t GetMaxHits(void) const { return _searchInfo._maxHits; } - uint32_t GetAlignedMaxHits(void) const { return _alignedMaxHits; } - uint32_t GetAlignedSearchOffset(void) const { return _alignedSearchOffset; } - const vespalib::string & GetRanking(void) const { return _ranking; } - uint32_t GetRandomSeed(void) const { return _randomSeed; } - uint32_t GetDateTime(void) const { return _dateTime; } - FastS_query &GetQuery(void) { return _query; } + bool AgeDropCheck(); + void DropResult(); + bool GotNoResultsYet() const { return _queryResult._hitbuf == NULL; } + uint32_t GetSearchOffset() const { return _searchInfo._searchOffset; } + uint32_t GetMaxHits() const { return _searchInfo._maxHits; } + uint32_t GetAlignedMaxHits() const { return _alignedMaxHits; } + uint32_t GetAlignedSearchOffset() const { return _alignedSearchOffset; } + const vespalib::string & GetRanking() const { return _ranking; } + uint32_t GetRandomSeed() const { return _randomSeed; } + uint32_t GetDateTime() const { return _dateTime; } + FastS_query &GetQuery() { return _query; } const char *GetSortSpec() const { return _query.GetSortSpec(); } const char *GetLocation() const { return _query.GetLocation(); } bool ShouldDropSortData() const { return _query.IsFlagSet(search::fs4transport::QFLAG_DROP_SORTDATA); } bool IsQueryFlagSet(uint32_t flag) const { return _query.IsFlagSet(flag); } - FastS_QueryResult *GetQueryResult(void) { + FastS_QueryResult *GetQueryResult() { return &_queryResult; } - FastS_DocsumsResult *GetDocsumsResult(void) { return &_docsumsResult; } - FastS_SearchInfo *GetSearchInfo(void) { return &_searchInfo; } + FastS_DocsumsResult *GetDocsumsResult() { return &_docsumsResult; } + FastS_SearchInfo *GetSearchInfo() { return &_searchInfo; } void SetStartTime(double timeref) { _startTime = timeref; } void AdjustSearchParameters(uint32_t partitions); void AdjustSearchParametersFinal(uint32_t partitions); void SetupQuery(uint32_t maxhits, uint32_t offset); - bool IsEstimate(void) const; - void ForceStrictLimits(void); - void InitEstimateMode(void); + bool IsEstimate() const; + void ForceStrictLimits(); + void InitEstimateMode(); double ElapsedSecs(double now) const { double ret = now - _startTime; if (ret < 0.0) @@ -95,7 +95,7 @@ public: } _alignedHitCount = alignedHitCount; } - void CalcHitCount(void) { + void CalcHitCount() { if (_alignedHitCount + _alignedSearchOffset > _searchInfo._searchOffset) { _queryResult._hitCount = _alignedHitCount + _alignedSearchOffset - _searchInfo._searchOffset; } else { @@ -105,7 +105,7 @@ public: _queryResult._hitCount = _searchInfo._maxHits; } } - void AllocAlignedHitBuf(void) { + void AllocAlignedHitBuf() { FastS_assert(_alignedHitBuf == NULL); if (_alignedHitCount != 0) { _alignedHitBuf = (FastS_hitresult*)malloc(sizeof(FastS_hitresult) * _alignedHitCount); @@ -134,11 +134,11 @@ public: } uint32_t *GetSortIndex() const { return _sortIndex; } char *GetSortData() const { return _sortData; } - FastS_hitresult *GetAlignedHitBuf(void) const { return _alignedHitBuf; } - FastS_hitresult *GetAlignedHitBufEnd(void) const { + FastS_hitresult *GetAlignedHitBuf() const { return _alignedHitBuf; } + FastS_hitresult *GetAlignedHitBufEnd() const { return _alignedHitBuf + _alignedHitCount; } - uint32_t GetAlignedHitCount(void) const { return _alignedHitCount; } + uint32_t GetAlignedHitCount() const { return _alignedHitCount; } void SetGroupResult(const char *groupResult) { _queryResult._groupResult = groupResult; } diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 83f81105f25..188757d69ba 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -1,19 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".groupingmanager"); + #include "groupingmanager.h" -#include <vector> -#include <vespa/searchlib/aggregation/grouping.h> -#include <vespa/searchlib/aggregation/groupinglevel.h> #include <vespa/searchlib/aggregation/fs4hit.h> -#include <vespa/vespalib/objects/objectpredicate.h> -#include <vespa/vespalib/objects/objectoperation.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchlib/expression/expressiontree.h> -#include <vespa/searchlib/expression/expressionnode.h> #include <vespa/searchcore/grouping/groupingsession.h> +#include <vespa/log/log.h> +LOG_SETUP(".groupingmanager"); + namespace search { namespace grouping { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index cdb7b01254e..f0e6520d0b4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -33,4 +33,5 @@ vespa_add_library(searchcore_attribute STATIC initialized_attributes_result.cpp sequential_attributes_initializer.cpp DEPENDS + searchcore_flushengine ) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp index b83680dc3f7..7d4ef067cbd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attribute_collection_spec_factory"); #include "attribute_collection_spec_factory.h" #include <vespa/searchlib/attribute/configconverter.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp index d55edb09ecf..9b2fc71702b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_directory.h" #include "attributedisklayout.h" #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> + #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_directory"); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp index d0ae670b4ba..54f2fb5ced5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attribute_factory"); - #include "attribute_factory.h" #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h index 92ad73cefdd..09127980434 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.h @@ -3,7 +3,6 @@ #pragma once #include "i_attribute_factory.h" -#include <set> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index 3f267d457dd..f77bddc73ea 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_manager_initializer.h" +#include "attributes_initializer_base.h" #include "attribute_collection_spec_factory.h" -#include "sequential_attributes_initializer.h" -#include <vespa/searchcore/proton/initializer/initializer_task.h> +#include <vespa/searchcorespi/index/i_thread_service.h> +#include <future> using search::AttributeVector; using search::GrowStrategy; @@ -33,7 +33,7 @@ public: _result(result) {} - virtual void run() override { + void run() override { AttributeInitializerResult result = _initializer->init(); if (result) { AttributesInitializerBase::considerPadAttribute(*result.getAttribute(), @@ -44,6 +44,51 @@ public: } }; +class AttributeManagerInitializerTask : public vespalib::Executor::Task +{ + std::promise<bool> _promise; + search::SerialNum _configSerialNum; + DocumentMetaStore::SP _documentMetaStore; + AttributeManager::SP _attrMgr; + InitializedAttributesResult &_attributesResult; + +public: + AttributeManagerInitializerTask(std::promise<bool> &&promise, + search::SerialNum configSerialNum, + DocumentMetaStore::SP documentMetaStore, + AttributeManager::SP attrMgr, + InitializedAttributesResult &attributesResult); + ~AttributeManagerInitializerTask() override; + void run() override; +}; + + +AttributeManagerInitializerTask::AttributeManagerInitializerTask(std::promise<bool> &&promise, + search::SerialNum configSerialNum, + DocumentMetaStore::SP documentMetaStore, + AttributeManager::SP attrMgr, + InitializedAttributesResult &attributesResult) + : _promise(std::move(promise)), + _configSerialNum(configSerialNum), + _documentMetaStore(documentMetaStore), + _attrMgr(attrMgr), + _attributesResult(attributesResult) +{ +} + +AttributeManagerInitializerTask::~AttributeManagerInitializerTask() +{ +} + +void +AttributeManagerInitializerTask::run() +{ + _attrMgr->addExtraAttribute(_documentMetaStore); + _attrMgr->addInitializedAttributes(_attributesResult.get()); + _attrMgr->pruneRemovedFields(_configSerialNum); + _promise.set_value(true); +} + class AttributeInitializerTasksBuilder : public IAttributeInitializerRegistry { private: @@ -101,6 +146,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN const GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, bool fastAccessAttributesOnly, + searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult) : _configSerialNum(configSerialNum), _documentMetaStore(documentMetaStore), @@ -109,6 +155,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN _attributeGrow(attributeGrow), _attributeGrowNumDocs(attributeGrowNumDocs), _fastAccessAttributesOnly(fastAccessAttributesOnly), + _master(master), _attributesResult(), _attrMgrResult(attrMgrResult) { @@ -121,9 +168,18 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN void AttributeManagerInitializer::run() { - _attrMgr->addExtraAttribute(_documentMetaStore); - _attrMgr->addInitializedAttributes(_attributesResult.get()); - _attrMgr->pruneRemovedFields(_configSerialNum); + std::promise<bool> promise; + std::future<bool> future = promise.get_future(); + /* + * Attribute manager and some its members (e.g. _attributeFieldWriter) assumes that work is performed + * by document db master thread and lacks locking to handle calls from multiple threads. + */ + _master.execute(std::make_unique<AttributeManagerInitializerTask>(std::move(promise), + _configSerialNum, + _documentMetaStore, + _attrMgr, + _attributesResult)); + (void) future.get(); *_attrMgrResult = _attrMgr; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index 9e1857d5f00..30d748932fd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -10,6 +10,8 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> +namespace searchcorespi { namespace index { class IThreadService; } } + namespace proton { /** @@ -25,6 +27,7 @@ private: search::GrowStrategy _attributeGrow; size_t _attributeGrowNumDocs; bool _fastAccessAttributesOnly; + searchcorespi::index::IThreadService &_master; InitializedAttributesResult _attributesResult; std::shared_ptr<AttributeManager::SP> _attrMgrResult; @@ -39,6 +42,7 @@ public: const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, bool fastAccessAttributesOnly, + searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult); virtual void run() override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp index 8d2665c0c3f..286c6b912ad 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attribute_populator"); #include "attribute_populator.h" - #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.attribute.attribute_populator"); + using search::IDestructorCallback; namespace proton { @@ -70,8 +69,10 @@ AttributePopulator::done() for (const auto &flushTarget : flushTargets) { assert(flushTarget->getFlushedSerialNum() < _configSerialNum); auto task = flushTarget->initFlush(_configSerialNum); - assert(task); - task->run(); + // shrink target only return task if able to shrink. + if (task) { + task->run(); + } assert(flushTarget->getFlushedSerialNum() == _configSerialNum); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp index 8c334293e63..e33a80824c5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_usage_filter.h" #include <sstream> namespace proton { -namespace -{ +namespace { void makeAddressSpaceMessage(std::ostream &os, const AddressSpaceUsageStats &usage) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index c4914580e34..fc5d366f14e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index 1854e243ae3..871a339005b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" #include <vespa/searchlib/attribute/attributevector.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h index 61565562232..a435875e3d5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h @@ -3,6 +3,7 @@ #pragma once #include "i_attribute_functor.h" +#include <memory> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp index aa26b6d81df..17d111db272 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_usage_stats.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index d0e59cdbda5..8e833ac8c67 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attribute_vector_explorer"); - #include "attribute_vector_explorer.h" #include <vespa/searchlib/attribute/enumstorebase.h> #include <vespa/searchlib/attribute/multi_value_mapping.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 56c941bc12d..a357fb92980 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -1,19 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.attributeadapter"); #include "attribute_writer.h" #include "attributemanager.h" #include <vespa/searchcore/proton/common/attrupdate.h> #include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/floatbase.h> -#include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> -#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> -#include <vespa/document/fieldvalue/arrayfieldvalue.h> -#include <type_traits> +#include <vespa/document/datatype/documenttype.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.attributeadapter"); using namespace document; using namespace search; @@ -119,6 +114,7 @@ applyReplayDone(uint32_t docIdLimit, AttributeVector &attr) { AttributeManager::padAttribute(attr, docIdLimit); attr.compactLidSpace(docIdLimit); + attr.shrinkLidSpace(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp index 349a8a632cb..69018d490e1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.attribute.attributedisklayout"); #include "attributedisklayout.h" -#include <vespa/vespalib/io/fileutil.h> #include "attribute_directory.h" +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/file.h> -namespace proton -{ +namespace proton { AttributeDiskLayout::AttributeDiskLayout(const vespalib::string &baseDir, PrivateConstructorTag) : _baseDir(baseDir), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h index 3b6a10e9e49..d68909df68f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h @@ -37,4 +37,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 2fa567e0c6d..5385063e17f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_factory.h" +#include "attribute_directory.h" #include "attributedisklayout.h" #include "attributemanager.h" #include "i_attribute_functor.h" @@ -8,9 +9,11 @@ #include "imported_attributes_repo.h" #include "sequential_attributes_initializer.h" #include "flushableattribute.h" +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> +#include <vespa/searchlib/common/threaded_compactable_lid_space.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -23,6 +26,7 @@ using search::AttributeContext; using search::AttributeEnumGuard; using search::AttributeGuard; using search::AttributeVector; +using search::common::ThreadedCompactableLidSpace; using search::TuneFileAttributes; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -60,6 +64,27 @@ bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Confi } } +search::SerialNum estimateShrinkSerialNum(const AttributeVector &attr) +{ + search::SerialNum serialNum = attr.getCreateSerialNum(); + if (serialNum > 0) { + --serialNum; + } + return std::max(attr.getStatus().getLastSyncToken(), serialNum); +} + +std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector::SP &attr, search::ISequencedTaskExecutor &attributeFieldWriter, AttributeDiskLayout &diskLayout) +{ + using Type = IFlushTarget::Type; + using Component = IFlushTarget::Component; + + const vespalib::string &name = attr->getName(); + auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(name)); + auto dir = diskLayout.createAttributeDir(name); + search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr); + return std::make_shared<ShrinkLidSpaceFlushTarget>("attribute.shrink." + name, Type::GC, Component::ATTRIBUTE, shrinkSerialNum, dir->getLastFlushTime(), shrinkwrap); +} + } AttributeManager::AttributeWrap::AttributeWrap(const AttributeVectorSP & a, @@ -91,6 +116,21 @@ AttributeManager::AttributeWrap::normalAttribute(const AttributeVectorSP &a) return AttributeWrap(a, false); } +AttributeManager::FlushableWrap::FlushableWrap() + : _flusher(), + _shrinker() +{ +} + +AttributeManager::FlushableWrap::FlushableWrap(FlushableAttributeSP flusher, ShrinkerSP shrinker) + : _flusher(std::move(flusher)), + _shrinker(std::move(shrinker)) +{ +} + +AttributeManager::FlushableWrap::~FlushableWrap() +{ +} AttributeVector::SP AttributeManager::internalAddAttribute(const AttributeSpec &spec, @@ -101,25 +141,24 @@ AttributeManager::internalAddAttribute(const AttributeSpec &spec, AttributeInitializerResult result = initializer.init(); if (result) { result.getAttribute()->setInterlock(_interlock); - addAttribute(AttributeWrap::normalAttribute(result.getAttribute())); + auto shrinker = allocShrinker(result.getAttribute(), _attributeFieldWriter, *_diskLayout); + addAttribute(AttributeWrap::normalAttribute(result.getAttribute()), shrinker); } return result.getAttribute(); } void -AttributeManager::addAttribute(const AttributeWrap &attribute) +AttributeManager::addAttribute(const AttributeWrap &attribute, const ShrinkerSP &shrinker) { LOG(debug, "Adding attribute vector '%s'", attribute.getAttribute()->getBaseFileName().c_str()); _attributes[attribute.getAttribute()->getName()] = attribute; assert(attribute.getAttribute()->getInterlock() == _interlock); if ( ! attribute.isExtra() ) { // Flushing of extra attributes is handled elsewhere - _flushables[attribute.getAttribute()->getName()] = FlushableAttribute::SP - (new FlushableAttribute(attribute.getAttribute(), _diskLayout->createAttributeDir(attribute.getAttribute()->getName()), - _tuneFileAttributes, - _fileHeaderContext, - _attributeFieldWriter, - _hwInfo)); + auto attr = attribute.getAttribute(); + const vespalib::string &name = attr->getName(); + auto flusher = std::make_shared<FlushableAttribute>(attr, _diskLayout->createAttributeDir(name), _tuneFileAttributes, _fileHeaderContext, _attributeFieldWriter, _hwInfo); + _flushables[attribute.getAttribute()->getName()] = FlushableWrap(flusher, shrinker); _writableAttributes.push_back(attribute.getAttribute().get()); } } @@ -133,11 +172,11 @@ AttributeManager::findAttribute(const vespalib::string &name) const : AttributeVector::SP(); } -FlushableAttribute::SP +const AttributeManager::FlushableWrap * AttributeManager::findFlushable(const vespalib::string &name) const { FlushableMap::const_iterator itr = _flushables.find(name); - return (itr != _flushables.end()) ? itr->second : FlushableAttribute::SP(); + return (itr != _flushables.end()) ? &itr->second : nullptr; } void @@ -150,7 +189,11 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, if (matchingTypes(av, aspec.getConfig())) { // transfer attribute LOG(debug, "Transferring attribute vector '%s' with %u docs and serial number %lu from current manager", av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken()); - addAttribute(AttributeWrap::normalAttribute(av)); + auto wrap = currMgr.findFlushable(aspec.getName()); + assert(wrap != nullptr); + auto shrinker = wrap->getShrinker(); + assert(shrinker); + addAttribute(AttributeWrap::normalAttribute(av), shrinker); } else { toBeAdded.push_back(aspec); } @@ -190,7 +233,7 @@ AttributeManager::transferExtraAttributes(const AttributeManager &currMgr) { for (const auto &kv : currMgr._attributes) { if (kv.second.isExtra()) { - addAttribute(kv.second); + addAttribute(kv.second, 0); } } } @@ -279,8 +322,10 @@ AttributeManager::addInitializedAttributes(const std::vector<AttributeInitialize { for (const auto &result : attributes) { assert(result); - result.getAttribute()->setInterlock(_interlock); - addAttribute(AttributeWrap::normalAttribute(result.getAttribute())); + auto attr = result.getAttribute(); + attr->setInterlock(_interlock); + auto shrinker = allocShrinker(attr, _attributeFieldWriter, *_diskLayout); + addAttribute(AttributeWrap::normalAttribute(attr), shrinker); } } @@ -288,15 +333,16 @@ void AttributeManager::addExtraAttribute(const AttributeVector::SP &attribute) { attribute->setInterlock(_interlock); - addAttribute(AttributeWrap::extraAttribute(attribute)); + addAttribute(AttributeWrap::extraAttribute(attribute), ShrinkerSP()); } void AttributeManager::flushAll(SerialNum currentSerial) { - for (const auto &kv : _flushables) { + auto flushTargets = getFlushTargets(); + for (const auto &ft : flushTargets) { vespalib::Executor::Task::UP task; - task = kv.second->initFlush(currentSerial); + task = ft->initFlush(currentSerial); if (task.get() != NULL) { task->run(); } @@ -306,7 +352,15 @@ AttributeManager::flushAll(SerialNum currentSerial) FlushableAttribute::SP AttributeManager::getFlushable(const vespalib::string &name) { - return findFlushable(name); + auto wrap = findFlushable(name); + return ((wrap != nullptr) ? wrap->getFlusher() : FlushableAttribute::SP()); +} + +AttributeManager::ShrinkerSP +AttributeManager::getShrinker(const vespalib::string &name) +{ + auto wrap = findFlushable(name); + return ((wrap != nullptr) ? wrap->getShrinker() : ShrinkerSP()); } size_t @@ -429,7 +483,8 @@ AttributeManager::getFlushTargets() const std::vector<IFlushTarget::SP> list; list.reserve(_flushables.size()); for (const auto &kv : _flushables) { - list.push_back(kv.second); + list.push_back(kv.second.getFlusher()); + list.push_back(kv.second.getShrinker()); } return list; } @@ -437,9 +492,12 @@ AttributeManager::getFlushTargets() const search::SerialNum AttributeManager::getFlushedSerialNum(const vespalib::string &name) const { - FlushableAttribute::SP flushable = findFlushable(name); - if (flushable.get() != nullptr) { - return flushable->getFlushedSerialNum(); + auto wrap = findFlushable(name); + if (wrap != nullptr) { + const auto &flusher = wrap->getFlusher(); + if (flusher) { + return flusher->getFlushedSerialNum(); + } } return 0; } @@ -449,7 +507,7 @@ AttributeManager::getOldestFlushedSerialNumber() const { SerialNum num = -1; for (const auto &kv : _flushables) { - num = std::min(num, kv.second->getFlushedSerialNum()); + num = std::min(num, kv.second.getFlusher()->getFlushedSerialNum()); } return num; } @@ -459,7 +517,7 @@ AttributeManager::getNewestFlushedSerialNumber() const { SerialNum num = 0; for (const auto &kv : _flushables) { - num = std::max(num, kv.second->getFlushedSerialNum()); + num = std::max(num, kv.second.getFlusher()->getFlushedSerialNum()); } return num; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index 7bfaf337420..4faa3011a1d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -35,6 +35,7 @@ namespace proton class AttributeDiskLayout; class FlushableAttribute; +class ShrinkLidSpaceFlushTarget; /** * Specialized attribute manager for proton. @@ -46,6 +47,7 @@ private: typedef search::SerialNum SerialNum; typedef AttributeCollectionSpec Spec; using FlushableAttributeSP = std::shared_ptr<FlushableAttribute>; + using ShrinkerSP = std::shared_ptr<ShrinkLidSpaceFlushTarget>; using IFlushTargetSP = std::shared_ptr<searchcorespi::IFlushTarget>; using AttributeVectorSP = std::shared_ptr<search::AttributeVector>; @@ -64,8 +66,20 @@ private: const AttributeVectorSP getAttribute() const { return _attr; } }; + class FlushableWrap + { + FlushableAttributeSP _flusher; + ShrinkerSP _shrinker; + public: + FlushableWrap(); + FlushableWrap(FlushableAttributeSP flusher, ShrinkerSP shrinker); + ~FlushableWrap(); + const FlushableAttributeSP &getFlusher() const { return _flusher; } + const ShrinkerSP &getShrinker() const { return _shrinker; } + }; + typedef vespalib::hash_map<vespalib::string, AttributeWrap> AttributeMap; - typedef vespalib::hash_map<vespalib::string, FlushableAttributeSP> FlushableMap; + typedef vespalib::hash_map<vespalib::string, FlushableWrap> FlushableMap; AttributeMap _attributes; FlushableMap _flushables; @@ -84,11 +98,11 @@ private: uint64_t serialNum, const IAttributeFactory &factory); - void addAttribute(const AttributeWrap &attribute); + void addAttribute(const AttributeWrap &attribute, const ShrinkerSP &shrinker); AttributeVectorSP findAttribute(const vespalib::string &name) const; - FlushableAttributeSP findFlushable(const vespalib::string &name) const; + const FlushableWrap *findFlushable(const vespalib::string &name) const; void transferExistingAttributes(const AttributeManager &currMgr, const Spec &newSpec, @@ -135,6 +149,8 @@ public: FlushableAttributeSP getFlushable(const vespalib::string &name); + ShrinkerSP getShrinker(const vespalib::string &name); + size_t getNumDocs() const; static void padAttribute(search::AttributeVector &v, uint32_t docIdLimit); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index 5d7a1526bf0..6f958ed9c3e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributesconfigscout.h" #include <vespa/searchlib/attribute/configconverter.h> using search::attribute::ConfigConverter; -namespace proton -{ +namespace proton { AttributesConfigScout::AttributesConfigScout(const AttributesConfig &live) : _live(live), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp index 930bf709e6c..c716dc12e85 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp @@ -1,14 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.exclusive_attribute_read_accessor"); - #include "exclusive_attribute_read_accessor.h" #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> -#include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/sync.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h index 2edecd1ca2a..cae1d1de770 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h @@ -2,9 +2,11 @@ #pragma once +#include <memory> + namespace search { -class AttributeVector; -class ISequencedTaskExecutor; + class AttributeVector; + class ISequencedTaskExecutor; } namespace vespalib { class Gate; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index f2f37fa5c16..e92a804b5de 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -13,21 +13,39 @@ namespace proton { namespace { -const vespalib::string FLUSH_TARGET_NAME_PREFIX("attribute."); +const vespalib::string FLUSH_TARGET_NAME_PREFIX("attribute.flush."); +const vespalib::string SHRINK_TARGET_NAME_PREFIX("attribute.shrink."); -bool isAttributeFlushTarget(const IFlushTarget::SP &flushTarget) { - const vespalib::string &targetName = flushTarget->getName(); - if ((flushTarget->getType() != IFlushTarget::Type::SYNC) || - (flushTarget->getComponent() != IFlushTarget::Component::ATTRIBUTE)) { - return false; +class FlushTargetFilter +{ + const vespalib::string &_prefix; + const IFlushTarget::Type _type; +public: + FlushTargetFilter(const vespalib::string &prefix, IFlushTarget::Type type) + : _prefix(prefix), + _type(type) + { } - return (targetName.substr(0, FLUSH_TARGET_NAME_PREFIX.size()) == FLUSH_TARGET_NAME_PREFIX); -} -vespalib::string attributeFlushTargetAttributeName(const IFlushTarget::SP &flushTarget) { - const vespalib::string &targetName = flushTarget->getName(); - return targetName.substr(FLUSH_TARGET_NAME_PREFIX.size()); -} + ~FlushTargetFilter() { } + + bool match(const IFlushTarget::SP &flushTarget) const { + const vespalib::string &targetName = flushTarget->getName(); + if ((flushTarget->getType() != _type) || + (flushTarget->getComponent() != IFlushTarget::Component::ATTRIBUTE)) { + return false; + } + return (targetName.substr(0, _prefix.size()) == _prefix); + } + + vespalib::string attributeName(const IFlushTarget::SP &flushTarget) { + const vespalib::string &targetName = flushTarget->getName(); + return targetName.substr(_prefix.size()); + } +}; + +FlushTargetFilter syncFilter(FLUSH_TARGET_NAME_PREFIX, IFlushTarget::Type::SYNC); +FlushTargetFilter shrinkFilter(SHRINK_TARGET_NAME_PREFIX, IFlushTarget::Type::GC); } @@ -72,8 +90,12 @@ FilterAttributeManager::getFlushTargets() const { std::vector<searchcorespi::IFlushTarget::SP> list; list.reserve(completeList.size()); for (const auto &flushTarget : completeList) { - if (isAttributeFlushTarget(flushTarget)) { - if (acceptAttribute(attributeFlushTargetAttributeName(flushTarget))) { + if (syncFilter.match(flushTarget)) { + if (acceptAttribute(syncFilter.attributeName(flushTarget))) { + list.push_back(flushTarget); + } + } else if (shrinkFilter.match(flushTarget)) { + if (acceptAttribute(shrinkFilter.attributeName(flushTarget))) { list.push_back(flushTarget); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index e3c27c752b0..d2e101436bf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -53,7 +53,7 @@ public: virtual void run() override; virtual SerialNum - getFlushSerial(void) const override + getFlushSerial() const override { return _syncToken; } @@ -70,9 +70,6 @@ FlushableAttribute::Flusher::Flusher(FlushableAttribute & fattr, SerialNum syncT fattr._attr->commit(syncToken, syncToken); AttributeVector &attr = *_fattr._attr; // Called by attribute field writer executor - if (attr.canShrinkLidSpace()) { - attr.shrinkLidSpace(); - } _flushFile = writer.getSnapshotDir(_syncToken) + "/" + attr.getName(); attr.setBaseFileName(_flushFile); _saver = attr.initSave(); @@ -171,7 +168,7 @@ FlushableAttribute::FlushableAttribute(const AttributeVectorSP attr, attributeFieldWriter, const HwInfo &hwInfo) : IFlushTarget(vespalib::make_string( - "attribute.%s", + "attribute.flush.%s", attr->getName().c_str()), Type::SYNC, Component::ATTRIBUTE), _attr(attr), @@ -202,30 +199,7 @@ IFlushTarget::MemoryGain FlushableAttribute::getApproxMemoryGain() const { int64_t used(_attr->getStatus().getUsed()); - int64_t canFree = 0; - if (_attr->canShrinkLidSpace()) { - uint32_t committedDocIdLimit = _attr->getCommittedDocIdLimit(); - uint32_t numDocs = _attr->getNumDocs(); - const attribute::Config &cfg = _attr->getConfig(); - if (committedDocIdLimit < numDocs) { - uint32_t elemSize = 4; - if (cfg.collectionType().isMultiValue()) { - if (cfg.huge()) { - elemSize = 8; - } - } else if (cfg.fastSearch()) { - // keep elemSize at 4 - } else { - elemSize = cfg.basicType().fixedSize(); - } - canFree = static_cast<int64_t>(elemSize) * - (numDocs - committedDocIdLimit); - if (canFree > used) - canFree = used; - } - } - return MemoryGain(used, - used - canFree); + return MemoryGain(used, used); } IFlushTarget::DiskGain diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp index 63ae28444d3..34ceaf60c90 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "imported_attributes_context.h" #include "imported_attributes_repo.h" #include <vespa/searchlib/attribute/attributeguard.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_repo.cpp b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_repo.cpp index d7a87609995..9261840f2b6 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_repo.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_repo.cpp @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "imported_attributes_repo.h" #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp index 4170a8e8be5..5470ab91008 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/sequential_attributes_initializer.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sequential_attributes_initializer.h" -#include "attributemanager.h" using search::AttributeVector; using search::SerialNum; diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp index 5236ebed261..569c49f0564 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bucket_db_explorer.h" #include <vespa/vespalib/data/slime/cursor.h> diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp index 4e417d91eac..056eced6bb4 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bucket_db_owner.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h index 4b26b1eaba1..c534cb5ae10 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h @@ -2,8 +2,8 @@ #pragma once -#include <mutex> #include "bucketdb.h" +#include <mutex> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index 6d2b3a0aede..7b5fabb1c74 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -1,32 +1,26 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.bucketdb"); #include "bucketdb.h" +#include <cassert> using document::GlobalId; using storage::spi::BucketChecksum; -namespace proton -{ - +namespace proton { -BucketDB::BucketDB(void) +BucketDB::BucketDB() : _map(), _cachedBucketId(), _cachedBucketState() { } - -BucketDB::~BucketDB(void) +BucketDB::~BucketDB() { checkEmpty(); clear(); } - bucketdb::BucketState * BucketDB::getBucketStatePtr(const BucketId &bucket) { @@ -37,7 +31,6 @@ BucketDB::getBucketStatePtr(const BucketId &bucket) return nullptr; } - void BucketDB::unloadBucket(const BucketId &bucket, const BucketState &delta) { @@ -46,7 +39,6 @@ BucketDB::unloadBucket(const BucketId &bucket, const BucketState &delta) *state -= delta; } - const bucketdb::BucketState & BucketDB::add(const GlobalId &gid, const BucketId &bucketId, @@ -70,7 +62,6 @@ BucketDB::remove(const GlobalId &gid, state.remove(gid, timestamp, docSize, subDbType); } - void BucketDB::modify(const GlobalId &gid, const BucketId &oldBucketId, @@ -161,20 +152,19 @@ BucketDB::getBuckets(BucketId::List &buckets) const } bool -BucketDB::empty(void) const +BucketDB::empty() const { return _map.empty(); } void -BucketDB::clear(void) +BucketDB::clear() { _map.clear(); } - void -BucketDB::checkEmpty(void) const +BucketDB::checkEmpty() const { for (auto &entry : _map) { const BucketState &state = entry.second; @@ -213,7 +203,6 @@ BucketDB::deleteEmptyBucket(const BucketId &bucketId) } } - void BucketDB::getActiveBuckets(BucketId::List &buckets) const { @@ -224,7 +213,6 @@ BucketDB::getActiveBuckets(BucketId::List &buckets) const } } - void BucketDB::populateActiveBuckets(const BucketId::List &buckets, BucketId::List &fixupBuckets) @@ -258,5 +246,4 @@ BucketDB::populateActiveBuckets(const BucketId::List &buckets, } } - } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index 9582713fc00..a51529eab1f 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -2,16 +2,12 @@ #pragma once -#include <vespa/document/base/globalid.h> +#include "bucketstate.h" #include <vespa/document/bucket/bucketid.h> -#include <vespa/persistence/spi/bucketinfo.h> -#include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/persistence/spi/result.h> -#include "bucketstate.h" #include <map> -namespace proton -{ +namespace proton { class BucketDB { @@ -31,11 +27,11 @@ private: BucketId _cachedBucketId; BucketState _cachedBucketState; - void clear(void); - void checkEmpty(void) const; + void clear(); + void checkEmpty() const; public: - BucketDB(void); - virtual ~BucketDB(void); + BucketDB(); + virtual ~BucketDB(); const BucketState & add(const GlobalId &gid, @@ -72,7 +68,7 @@ public: BucketState cachedGet(const BucketId &bucketId) const; bool hasBucket(const BucketId &bucketId) const; void getBuckets(BucketId::List & buckets) const; - bool empty(void) const; + bool empty() const; void setBucketState(const BucketId &bucketId, bool active); void createBucket(const BucketId &bucketId); void deleteEmptyBucket(const BucketId &bucketId); diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp index a36fca85def..2160ce4be78 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.cpp @@ -1,20 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "bucketdb.h" -#include "bucketsessionbase.h" +#include "bucketdbhandler.h" #include "splitbucketsession.h" #include "joinbucketssession.h" -#include <vespa/searchlib/common/serialnum.h> -#include "bucketdbhandler.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> -namespace proton -{ - -namespace bucketdb -{ - +namespace proton::bucketdb { BucketDBHandler::BucketDBHandler(BucketDBOwner &bucketDB) : _bucketDB(bucketDB), @@ -22,7 +13,6 @@ BucketDBHandler::BucketDBHandler(BucketDBOwner &bucketDB) { } - void BucketDBHandler::addDocumentMetaStore(IDocumentMetaStore *dms, search::SerialNum flushedSerialNum) @@ -30,7 +20,6 @@ BucketDBHandler::addDocumentMetaStore(IDocumentMetaStore *dms, _dmsv.push_back(MetaStoreDesc(dms, flushedSerialNum)); } - void BucketDBHandler::handleSplit(search::SerialNum serialNum, const BucketId &source, @@ -87,21 +76,17 @@ BucketDBHandler::handleJoin(search::SerialNum serialNum, session.finish(); } - void BucketDBHandler::handleCreateBucket(const BucketId &bucketId) { _bucketDB.takeGuard()->createBucket(bucketId); } - void BucketDBHandler::handleDeleteBucket(const BucketId &bucketId) { _bucketDB.takeGuard()->deleteEmptyBucket(bucketId); } - } -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h index 464cb2e69a1..7d855abbaae 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdbhandler.h @@ -6,11 +6,7 @@ #include "ibucketdbhandler.h" #include "ibucketdbhandlerinitializer.h" -namespace proton -{ - -namespace bucketdb -{ +namespace proton::bucketdb { /** * The BucketDBHandler class handles operations on a bucket db. @@ -65,5 +61,3 @@ public: }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h index f0db5e17867..fdc77f6b2a7 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdeltapair.h @@ -2,11 +2,9 @@ #pragma once -namespace proton -{ +#include "bucketstate.h" -namespace bucketdb -{ +namespace proton::bucketdb { /** * Class BucketDeltaPair represent the deltas to bucket states caused by @@ -26,5 +24,3 @@ public: }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp index 4173cc28f5d..b72fe8e9934 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.cpp @@ -1,14 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "bucketdb.h" #include "bucketsessionbase.h" -namespace proton -{ - -namespace bucketdb -{ +namespace proton::bucketdb { BucketSessionBase::BucketSessionBase(BucketDBOwner &bucketDB) : _bucketDB(bucketDB.takeGuard()) @@ -39,7 +33,4 @@ BucketSessionBase::calcFixupNeed(BucketState *state, bool wantActive, return false; } - -} - } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h index 64983ae45bf..5fe335ad59c 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketsessionbase.h @@ -4,11 +4,7 @@ #include "bucket_db_owner.h" -namespace proton -{ - -namespace bucketdb -{ +namespace proton::bucketdb { /** * Base class for split/join handling utility classes that bundles temporary @@ -27,14 +23,9 @@ protected: public: BucketSessionBase(BucketDBOwner &bucketDB); - bool - extractInfo(const BucketId &bucket, BucketState *&info); + bool extractInfo(const BucketId &bucket, BucketState *&info); - static bool - calcFixupNeed(BucketState *state, bool wantActive, bool fixup); + static bool calcFixupNeed(BucketState *state, bool wantActive, bool fixup); }; - -} - } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp index e8fad2054da..45d1b2957e3 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp @@ -1,17 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "bucketstate.h" +#include <cassert> -#include "bucketdb.h" +namespace proton::bucketdb { -namespace proton -{ - -namespace bucketdb -{ - -namespace -{ +namespace { uint32_t gidChecksum(const document::GlobalId &gid) @@ -104,7 +98,7 @@ BucketState::modify(const Timestamp &oldTimestamp, uint32_t oldDocSize, bool -BucketState::empty(void) const +BucketState::empty() const { if (getReadyCount() != 0 || getRemovedCount() != 0 || getNotReadyCount() != 0) @@ -181,7 +175,5 @@ BucketState::operator storage::spi::BucketInfo() const _active ? BucketInfo::ACTIVE : BucketInfo::NOT_ACTIVE); } - } -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h index b2b4760cb8b..50f862e1e41 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h @@ -2,11 +2,11 @@ #pragma once -namespace proton -{ +#include <vespa/searchcore/proton/common/subdbtype.h> +#include <vespa/document/base/globalid.h> +#include <vespa/persistence/spi/bucketinfo.h> -namespace bucketdb -{ +namespace proton::bucketdb { /** * Class BucketState represent the known state of a bucket in raw form. @@ -34,85 +34,33 @@ public: static uint32_t calcChecksum(const GlobalId &gid, const Timestamp ×tamp); - void - add(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType); - - void - remove(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType); + void add(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType); + void remove(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSize, SubDbType subDbType); - void - modify(const Timestamp &oldTimestamp, uint32_t oldDocSize, - const Timestamp &newTimestamp, uint32_t newDocSize, - SubDbType subDbType); + void modify(const Timestamp &oldTimestamp, uint32_t oldDocSize, + const Timestamp &newTimestamp, uint32_t newDocSize, + SubDbType subDbType); - bool - isActive() const - { - return _active; - } - - BucketState & - setActive(bool active) - { + BucketState &setActive(bool active) { _active = active; return *this; } - uint32_t - getReadyCount() const - { - return _docCount[READY]; - } - - uint32_t - getRemovedCount() const - { - return _docCount[REMOVED]; - } - - uint32_t - getNotReadyCount() const - { - return _docCount[NOTREADY]; - } - + bool isActive() const { return _active; } + uint32_t getReadyCount() const { return _docCount[READY]; } + uint32_t getRemovedCount() const { return _docCount[REMOVED]; } + uint32_t getNotReadyCount() const { return _docCount[NOTREADY]; } size_t getReadyDocSizes() const { return _docSizes[READY]; } size_t getRemovedDocSizes() const { return _docSizes[REMOVED]; } size_t getNotReadyDocSizes() const { return _docSizes[NOTREADY]; } - - uint32_t - getDocumentCount() const - { - return getReadyCount() + getNotReadyCount(); - } - - uint32_t - getEntryCount() const - { - return getDocumentCount() + getRemovedCount(); - } - - storage::spi::BucketChecksum - getChecksum() const - { - return storage::spi::BucketChecksum(_checksum); - } - - bool - empty() const; - - BucketState & - operator+=(const BucketState &rhs); - - BucketState & - operator-=(const BucketState &rhs); - - void - applyDelta(BucketState *src, BucketState *dst) const; - + uint32_t getDocumentCount() const { return getReadyCount() + getNotReadyCount(); } + uint32_t getEntryCount() const { return getDocumentCount() + getRemovedCount(); } + storage::spi::BucketChecksum getChecksum() const { return storage::spi::BucketChecksum(_checksum); } + bool empty() const; + BucketState &operator+=(const BucketState &rhs); + BucketState &operator-=(const BucketState &rhs); + void applyDelta(BucketState *src, BucketState *dst) const; operator storage::spi::BucketInfo() const; }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp index ce46df2c450..60024fc63ac 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.cpp @@ -1,16 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "bucketdb.h" -#include "bucketsessionbase.h" #include "joinbucketssession.h" #include "bucketdeltapair.h" +#include <cassert> -namespace proton -{ - -namespace bucketdb -{ +namespace proton::bucketdb { JoinBucketsSession::JoinBucketsSession(BucketDBOwner &bucketDB, const BucketId &source1, @@ -107,7 +101,5 @@ JoinBucketsSession::finish() } } - } -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h index f05521a06c0..bdb43130e19 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/joinbucketssession.h @@ -2,11 +2,9 @@ #pragma once -namespace proton -{ +#include "bucketsessionbase.h" -namespace bucketdb -{ +namespace proton::bucketdb { class BucketDeltaPair; @@ -82,5 +80,3 @@ public: }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp index 5cc4d74cf25..389ec594878 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.cpp @@ -1,16 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include "bucketdb.h" -#include "bucketsessionbase.h" #include "splitbucketsession.h" #include "bucketdeltapair.h" +#include <cassert> -namespace proton -{ - -namespace bucketdb -{ +namespace proton::bucketdb { SplitBucketSession::SplitBucketSession(BucketDBOwner &bucketDB, @@ -93,7 +87,5 @@ SplitBucketSession::finish() } } - } -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h index 260a32d7dbd..8260c99547d 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/splitbucketsession.h @@ -2,11 +2,9 @@ #pragma once -namespace proton -{ +#include "bucketsessionbase.h" -namespace bucketdb -{ +namespace proton::bucketdb { class BucketDeltaPair; @@ -32,8 +30,7 @@ private: BucketId _target1; BucketId _target2; - void - applyDelta(const BucketState &delta, BucketState *src, BucketId &dstBucket); + void applyDelta(const BucketState &delta, BucketState *src, BucketId &dstBucket); public: SplitBucketSession(BucketDBOwner &bucketDB, @@ -44,62 +41,28 @@ public: /* * Reflect move of documents to target1 and target2 in bucket states */ - void - applyDeltas(const BucketDeltaPair &deltas); - - bool - getSourceActive() const - { - return _sourceActive; - } + void applyDeltas(const BucketDeltaPair &deltas); + bool getSourceActive() const { return _sourceActive; } /* * Return true if bitvector for active lids need to be adjusted in * document meta store due to old documents in target1 and active * state change. */ - bool - mustFixupTarget1ActiveLids() const - { - return _adjustTarget1ActiveLids; - } + bool mustFixupTarget1ActiveLids() const { return _adjustTarget1ActiveLids; } /* * Return true if bitvector for active lids need to be adjusted in * document meta store due to old documents in target2 and active * state change. */ - bool - mustFixupTarget2ActiveLids() const - { - return _adjustTarget2ActiveLids; - } - - void - setup(); - - void - finish(); + bool mustFixupTarget2ActiveLids() const { return _adjustTarget2ActiveLids; } - const BucketId & - getSource() const - { - return _source; - } - - const BucketId & - getTarget1() const - { - return _target1; - } - - const BucketId & - getTarget2() const - { - return _target2; - } + void setup(); + void finish(); + const BucketId &getSource() const { return _source; } + const BucketId &getTarget1() const { return _target1; } + const BucketId &getTarget2() const { return _target2; } }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index d89eb91cf6f..48c87616b9b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -15,7 +15,6 @@ vespa_add_library(searchcore_pcommon STATIC hw_info_sampler.cpp indexschema_inspector.cpp monitored_refcount.cpp - schemautil.cpp selectpruner.cpp selectcontext.cpp state_reporter_utils.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 73df7d511c9..10c2adfd1f2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.attributefieldvaluenode"); #include "attributefieldvaluenode.h" #include "selectcontext.h" -#include <vespa/document/select/value.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/attribute/attributevector.h> -namespace proton -{ + +namespace proton { using document::select::Context; using document::select::FloatValue; @@ -25,11 +22,10 @@ using search::attribute::BasicType; using search::attribute::CollectionType; using search::attribute::IAttributeVector; - AttributeFieldValueNode:: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const search::AttributeVector::SP &attribute) + const std::shared_ptr<search::AttributeVector> &attribute) : FieldValueNode(doctype, field), _attribute(attribute) { @@ -99,5 +95,11 @@ AttributeFieldValueNode::traceValue(const Context &context, } +document::select::ValueNode::UP +AttributeFieldValueNode::clone() const +{ + return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute)); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h index a5d35eebad2..c0ffbaea987 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h @@ -1,38 +1,24 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/select/valuenode.h> -#include <vespa/searchlib/attribute/attributevector.h> - -namespace proton -{ +#include <vespa/document/select/valuenodes.h> +namespace search { class AttributeVector; } +namespace proton { class AttributeFieldValueNode : public document::select::FieldValueNode { - search::AttributeVector::SP _attribute; + using Context = document::select::Context; + std::shared_ptr<search::AttributeVector> _attribute; public: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const search::AttributeVector::SP &attribute); - - virtual std::unique_ptr<document::select::Value> - getValue(const document::select::Context &context) const override; + const std::shared_ptr<search::AttributeVector> &attribute); - virtual std::unique_ptr<document::select::Value> - traceValue(const document::select::Context &context, - std::ostream& out) const override; - - document::select::ValueNode::UP - clone(void) const override - { - return wrapParens(new AttributeFieldValueNode(getDocType(), - getFieldName(), - _attribute)); - } + std::unique_ptr<document::select::Value> getValue(const Context &context) const override; + std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override; + document::select::ValueNode::UP clone() const override; }; } // namespace proton - - diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp index d54603fd898..51cb9a17b5b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp @@ -20,6 +20,7 @@ #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/changevector.hpp> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".attrupdate"); diff --git a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp index bdf35a09038..f5acc578792 100644 --- a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp @@ -1,11 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.bucketfactory"); - #include "bucketfactory.h" -#include <persistence/spi/types.h> using document::BucketId; using document::DocumentId; @@ -29,5 +24,4 @@ BucketFactory::getBucket(const DocumentId &docId) return Bucket(getBucketId(docId), PartitionId(0)); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index 934737122e7..9a15c257dcc 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cachedselect.h" -#include <vespa/document/select/valuenode.h> -#include <vespa/document/select/cloningvisitor.h> #include "attributefieldvaluenode.h" +#include "selectpruner.h" #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/document/select/parser.h> -#include "selectpruner.h" #include <vespa/log/log.h> LOG_SETUP(".proton.common.cachedselect"); @@ -85,7 +84,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) _valueNode = expr.clone(); return; } - AttributeVector::SP av(ag->getSP()); + std::shared_ptr<search::AttributeVector> av(ag->getSP()); if (av->getCollectionType() == CollectionType::SINGLE) { ++_svAttrs; AttrMap::iterator it(_amap.find(name)); @@ -113,7 +112,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) } -CachedSelect::CachedSelect(void) +CachedSelect::CachedSelect() : _attributes(), _select(), _fieldNodes(0u), diff --git a/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp index 6acd99d5285..fa14ed4ce53 100644 --- a/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/commit_time_tracker.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "commit_time_tracker.h" namespace proton { @@ -29,8 +29,6 @@ CommitTimeTracker::needCommit() const return true; } - - void CommitTimeTracker::setVisibilityDelay(fastos::TimeStamp visibilityDelay) { diff --git a/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp index ae9911289db..4cf8eafcf41 100644 --- a/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/dbdocumentid.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.dbdocumentid"); #include "dbdocumentid.h" diff --git a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp index 0cf2e97fdfb..7e86cae6946 100644 --- a/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/document_type_inspector.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.document_type_inspector"); - #include "document_type_inspector.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp index cb3963b4790..63ee323d078 100644 --- a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp @@ -1,14 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.eventlogger"); #include "eventlogger.h" #include <vespa/searchlib/util/logutil.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.common.eventlogger"); + using search::util::LogUtil; using vespalib::JSONStringer; +using vespalib::make_string; namespace { @@ -16,10 +17,7 @@ using search::SerialNum; using vespalib::string; void -doTransactionLogReplayStart(const string &domainName, - SerialNum first, - SerialNum last, - const string &eventName) +doTransactionLogReplayStart(const string &domainName, SerialNum first, SerialNum last, const string &eventName) { JSONStringer jstr; jstr.beginObject(); @@ -34,9 +32,7 @@ doTransactionLogReplayStart(const string &domainName, } void -doTransactionLogReplayComplete(const string &domainName, - int64_t elapsedTimeMs, - const string &eventName) +doTransactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs, const string &eventName) { JSONStringer jstr; jstr.beginObject(); @@ -51,22 +47,14 @@ doTransactionLogReplayComplete(const string &domainName, namespace proton { void -EventLogger::transactionLogReplayStart(const string &domainName, - SerialNum first, - SerialNum last) +EventLogger::transactionLogReplayStart(const string &domainName, SerialNum first, SerialNum last) { - doTransactionLogReplayStart(domainName, - first, - last, - "transactionlog.replay.start"); + doTransactionLogReplayStart(domainName, first, last, "transactionlog.replay.start"); } void -EventLogger::transactionLogReplayProgress(const string &domainName, - float progress, - SerialNum first, - SerialNum last, - SerialNum current) +EventLogger::transactionLogReplayProgress(const string &domainName, float progress, + SerialNum first, SerialNum last, SerialNum current) { JSONStringer jstr; jstr.beginObject(); @@ -85,9 +73,7 @@ EventLogger::transactionLogReplayProgress(const string &domainName, void EventLogger::transactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs) { - doTransactionLogReplayComplete(domainName, - elapsedTimeMs, - "transactionlog.replay.complete"); + doTransactionLogReplayComplete(domainName, elapsedTimeMs, "transactionlog.replay.complete"); } void @@ -101,12 +87,8 @@ EventLogger::flushInit(const string &name) } void -EventLogger::flushStart(const string &name, - int64_t beforeMemory, - int64_t afterMemory, - int64_t toFreeMemory, - SerialNum unflushed, - SerialNum current) +EventLogger::flushStart(const string &name, int64_t beforeMemory, int64_t afterMemory, + int64_t toFreeMemory, SerialNum unflushed, SerialNum current) { JSONStringer jstr; jstr.beginObject(); @@ -127,17 +109,17 @@ EventLogger::flushStart(const string &name, } void -EventLogger::flushComplete(const string &name, - int64_t elapsedTimeMs, - const string &outputPath, - size_t outputPathElems) +EventLogger::flushComplete(const string &name, int64_t elapsedTimeMs, + const string &outputPath, size_t outputPathElems) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("name").appendString(name); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); - jstr.appendKey("output"); - LogUtil::logDir(jstr, outputPath, outputPathElems); + if (!outputPath.empty()) { + jstr.appendKey("output"); + LogUtil::logDir(jstr, outputPath, outputPathElems); + } jstr.endObject(); EV_STATE("flush.complete", jstr.toString().c_str()); } @@ -281,7 +263,7 @@ loadComponentStart(const vespalib::string &subDbName, const vespalib::string &co jstr.beginObject(); jstr.appendKey("documentsubdb").appendString(subDbName); jstr.endObject(); - EV_STATE(vespalib::make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().c_str()); + EV_STATE(make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().c_str()); } void @@ -292,7 +274,7 @@ loadComponentComplete(const vespalib::string &subDbName, const vespalib::string jstr.appendKey("documentsubdb").appendString(subDbName); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); jstr.endObject(); - EV_STATE(vespalib::make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().c_str()); + EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().c_str()); } } diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp index 9d0a63677e0..2e701d20c62 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp @@ -1,13 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.feedtoken"); #include "feedtoken.h" #include <vespa/vespalib/util/atomic.h> #include <vespa/searchcore/proton/metrics/feed_metrics.h> - namespace proton { FeedToken::FeedToken(ITransport &transport, mbus::Reply::UP reply) : @@ -77,7 +73,7 @@ FeedToken::State::ack(const FeedOperation::Type opType, void -FeedToken::State::incNeededAcks(void) +FeedToken::State::incNeededAcks() { assert(_reply.get() != NULL); uint32_t prev(_unAckedCount++); diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h index fee84d53ce5..b73f7bc7faa 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h @@ -43,7 +43,7 @@ private: void ack(const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics); - void incNeededAcks(void); + void incNeededAcks(); void fail(uint32_t errNum, const vespalib::string &errMsg); void trace(uint32_t traceLevel, const vespalib::string &traceMsg); @@ -95,7 +95,7 @@ public: } void - incNeededAcks(void) const + incNeededAcks() const { _state->incNeededAcks(); } diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index 341a4d77f95..4d9ce09d77d 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -1,13 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcore/proton/common/doctypename.h> +#include "doctypename.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <map> +#include <vector> namespace proton { @@ -148,7 +149,7 @@ public: * Clear all handlers. */ void - clear(void) + clear() { _handlers.clear(); } diff --git a/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp b/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp deleted file mode 100644 index 2fcf4a11c9b..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/schemautil.cpp +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "schemautil.h" - -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.schemautil"); - -using namespace search::index; - -namespace proton { - -namespace { - -class FieldQuad -{ -public: - vespalib::string _name; - vespalib::string _dataType; - vespalib::string _collectionType; - vespalib::string _location; - - FieldQuad(const vespalib::string &name, - const vespalib::string &dataType, - const vespalib::string &collectionType, - const vespalib::string &location); - ~FieldQuad(); - - bool - operator<(const FieldQuad &rhs) const - { - if (_name != rhs._name) - return _name < rhs._name; - if (_dataType != rhs._dataType) - return _dataType < rhs._dataType; - if (_collectionType != rhs._collectionType) - return _collectionType < rhs._collectionType; - return _location < rhs._location; - } -}; - -FieldQuad::FieldQuad(const vespalib::string &name, - const vespalib::string &dataType, - const vespalib::string &collectionType, - const vespalib::string &location) - : _name(name), - _dataType(dataType), - _collectionType(collectionType), - _location(location) -{ -} -FieldQuad::~FieldQuad() {} - -} - - -Schema::SP -SchemaUtil::makeHistorySchema(const Schema &newSchema, - const Schema &oldSchema, - const Schema &oldHistory) -{ - return makeHistorySchema(newSchema, oldSchema, oldHistory, - fastos::ClockSystem::now()); -} - -Schema::SP -SchemaUtil::makeHistorySchema(const Schema &newSchema, - const Schema &oldSchema, - const Schema &oldHistory, - int64_t now) -{ - Schema::SP history(new Schema); - - uint32_t fields = oldHistory.getNumIndexFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - const Schema::IndexField &field = oldHistory.getIndexField(fieldId); - uint32_t nFieldId = newSchema.getIndexFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field not re-added, keep it in history - history->addIndexField(field); - } - } - fields = oldHistory.getNumAttributeFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - const Schema::AttributeField &field = - oldHistory.getAttributeField(fieldId); - uint32_t nFieldId = - newSchema.getAttributeFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field not re-added, keep it in history - history->addAttributeField(field); - } - } - fields = oldHistory.getNumSummaryFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - const Schema::SummaryField &field = - oldHistory.getSummaryField(fieldId); - uint32_t nFieldId = - newSchema.getSummaryFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field not re-added, keep it in history - history->addSummaryField(field); - } - } - fields = oldSchema.getNumIndexFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - Schema::IndexField field = oldSchema.getIndexField(fieldId); - uint32_t nFieldId = newSchema.getIndexFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field removed, add to history - uint32_t oFieldId = history->getIndexFieldId(field.getName()); - if (oFieldId == Schema::UNKNOWN_FIELD_ID) { - field.setTimestamp(now); - history->addIndexField(field); - } - } - } - fields = oldSchema.getNumAttributeFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - Schema::AttributeField field = oldSchema.getAttributeField(fieldId); - uint32_t nFieldId = - newSchema.getAttributeFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field removed, add to history - uint32_t oFieldId = history->getAttributeFieldId(field.getName()); - if (oFieldId == Schema::UNKNOWN_FIELD_ID) { - field.setTimestamp(now); - history->addAttributeField(field); - } - } - } - fields = oldSchema.getNumSummaryFields(); - for (uint32_t fieldId = 0; fieldId < fields; ++fieldId) { - Schema::SummaryField field = oldSchema.getSummaryField(fieldId); - uint32_t nFieldId = - newSchema.getSummaryFieldId(field.getName()); - if (nFieldId == Schema::UNKNOWN_FIELD_ID) { - // Field removed, add to history - uint32_t oFieldId = history->getSummaryFieldId(field.getName()); - if (oFieldId == Schema::UNKNOWN_FIELD_ID) { - field.setTimestamp(now); - history->addSummaryField(field); - } - } - } - return history; -} - - -void -SchemaUtil::listSchema(const Schema &schema, - std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations) -{ - std::vector<FieldQuad> quads; - for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) { - const Schema::AttributeField &field = schema.getAttributeField(i); - quads.push_back( - FieldQuad(field.getName(), - schema::getTypeName(field.getDataType()), - schema::getTypeName(field.getCollectionType()), - "a")); - } - for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) { - const Schema::IndexField &field = schema.getIndexField(i); - quads.push_back( - FieldQuad(field.getName(), - schema::getTypeName(field.getDataType()), - schema::getTypeName(field.getCollectionType()), - "i")); - } - for (uint32_t i = 0; i < schema.getNumSummaryFields(); ++i) { - const Schema::SummaryField &field = schema.getSummaryField(i); - quads.push_back( - FieldQuad(field.getName(), - schema::getTypeName(field.getDataType()), - schema::getTypeName(field.getCollectionType()), - "s")); - } - std::sort(quads.begin(), quads.end()); - std::string name; - std::string dataType; - std::string collectionType; - std::string location; - for (std::vector<FieldQuad>::const_iterator - it = quads.begin(), ite = quads.end(); it != ite; ++it) { - if (it->_name != name || it->_dataType != dataType || - it->_collectionType != collectionType) { - if (!name.empty()) { - fieldNames.push_back(name); - fieldDataTypes.push_back(dataType); - fieldCollectionTypes.push_back(collectionType); - fieldLocations.push_back(location); - } - name = it->_name; - dataType = it->_dataType; - collectionType = it->_collectionType; - location.clear(); - } - location += it->_location; - } - if (!name.empty()) { - fieldNames.push_back(name); - fieldDataTypes.push_back(dataType); - fieldCollectionTypes.push_back(collectionType); - fieldLocations.push_back(location); - } -} - - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/schemautil.h b/searchcore/src/vespa/searchcore/proton/common/schemautil.h deleted file mode 100644 index 36cc952f921..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/schemautil.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchcommon/common/schema.h> -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace proton -{ - -class SchemaUtil -{ -public: - /** - * Make new history schema based on new and old schema and old history. - */ - static search::index::Schema::SP - makeHistorySchema(const search::index::Schema &newSchema, - const search::index::Schema &oldSchema, - const search::index::Schema &oldHistory); - - static search::index::Schema::SP - makeHistorySchema(const search::index::Schema &newSchema, - const search::index::Schema &oldSchema, - const search::index::Schema &oldHistory, - int64_t timestamp); - - /** - * Iterate through the given schema and fill out the given string vectors. - */ - static void - listSchema(const search::index::Schema &schema, - std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations); -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp index 36fa242daec..1e8a15593c0 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp @@ -29,7 +29,7 @@ SelectContext::SelectContext(const CachedSelect &cachedSelect) SelectContext::~SelectContext() { } void -SelectContext::getAttributeGuards(void) +SelectContext::getAttributeGuards() { _guards->resize(_cachedSelect._attributes.size()); auto j(_cachedSelect._attributes.begin()); @@ -40,7 +40,7 @@ SelectContext::getAttributeGuards(void) void -SelectContext::dropAttributeGuards(void) +SelectContext::dropAttributeGuards() { _guards->clear(); } diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp index 12acfdcfd7e..7e92be7507e 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp @@ -1,22 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.selectpruner"); #include "selectpruner.h" #include <vespa/document/base/exceptions.h> -#include <vespa/document/base/fieldpath.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/compare.h> -#include <vespa/document/select/operator.h> #include <vespa/document/select/constant.h> #include <vespa/document/select/branch.h> #include <vespa/document/select/doctype.h> #include <vespa/document/select/invalidconstant.h> -#include <vespa/document/select/constant.h> +#include <vespa/document/select/valuenodes.h> #include <vespa/searchlib/attribute/iattributemanager.h> - using document::select::And; using document::select::Compare; using document::select::Constant; @@ -49,8 +44,7 @@ using document::Field; using document::FieldNotFoundException; using search::AttributeGuard; -namespace proton -{ +namespace proton { SelectPrunerBase::SelectPrunerBase(const vespalib::string &docType, const search::IAttributeManager *amgr, @@ -439,7 +433,7 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) void -SelectPruner::invertNode(void) +SelectPruner::invertNode() { _resultSet = _resultSet.calcNot(); if (isInvalid()) { @@ -498,7 +492,7 @@ SelectPruner::addNodeCount(const SelectPruner &rhs) void -SelectPruner::setInvalidVal(void) +SelectPruner::setInvalidVal() { _constVal = true; _priority = InvalidValPriority; @@ -507,7 +501,7 @@ SelectPruner::setInvalidVal(void) void -SelectPruner::setInvalidConst(void) +SelectPruner::setInvalidConst() { _constVal = true; _priority = InvalidConstPriority; @@ -542,7 +536,7 @@ SelectPruner::resolveTernaryConst(bool wantInverted) bool -SelectPruner::isFalse(void) const +SelectPruner::isFalse() const { if (!_constVal) return false; @@ -560,7 +554,7 @@ SelectPruner::isFalse(void) const bool -SelectPruner::isTrue(void) const +SelectPruner::isTrue() const { if (!_constVal) return false; @@ -578,7 +572,7 @@ SelectPruner::isTrue(void) const bool -SelectPruner::isInvalid(void) const +SelectPruner::isInvalid() const { if (!_constVal) return false; @@ -596,7 +590,7 @@ SelectPruner::isInvalid(void) const bool -SelectPruner::isInvalidVal(void) const +SelectPruner::isInvalidVal() const { if (!_constVal) return false; @@ -606,7 +600,7 @@ SelectPruner::isInvalidVal(void) const bool -SelectPruner::isNullVal(void) const +SelectPruner::isNullVal() const { if (!_constVal) return false; @@ -616,7 +610,7 @@ SelectPruner::isNullVal(void) const bool -SelectPruner::isConst(void) const +SelectPruner::isConst() const { return _constVal; } diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h index 92bbb435349..7b6c9954124 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h @@ -9,8 +9,9 @@ namespace search { class IAttributeManager; } -namespace proton -{ +namespace document { class DocumentTypeRepo; } + +namespace proton { class SelectPrunerBase { @@ -50,100 +51,36 @@ public: bool hasFields); SelectPruner(const SelectPruner *rhs); - - virtual - ~SelectPruner(void); - - uint32_t - getFieldNodes(void) const - { - return _fieldNodes; - } - - uint32_t - getAttrFieldNodes(void) const - { - return _attrFieldNodes; - } - - const document::select::ResultSet & - getResultSet(void) const - { - return _resultSet; - } - - bool - isFalse(void) const; - - bool - isTrue(void) const; - - bool - isInvalid(void) const; - - bool - isConst(void) const; - - void - trace(std::ostream &t); - - void - process(const document::select::Node &node); + virtual ~SelectPruner(); + + uint32_t getFieldNodes() const { return _fieldNodes; } + uint32_t getAttrFieldNodes() const { return _attrFieldNodes; } + const document::select::ResultSet & getResultSet() const { return _resultSet; } + bool isFalse() const; + bool isTrue() const; + bool isInvalid() const; + bool isConst() const; + void trace(std::ostream &t); + void process(const document::select::Node &node); private: - virtual void - visitAndBranch(const document::select::And &expr) override; - - virtual void - visitComparison(const document::select::Compare &expr) override; - - virtual void - visitDocumentType(const document::select::DocType &expr) override; - - virtual void - visitNotBranch(const document::select::Not &expr) override; - - virtual void - visitOrBranch(const document::select::Or &expr) override; - - virtual void - visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override; - - virtual void - visitFunctionValueNode(const document::select::FunctionValueNode &expr) override; - - virtual void - visitFieldValueNode(const document::select::FieldValueNode &expr) override; - - void - invertNode(void); - - const document::select::Operator & - getOperator(const document::select::Operator &op); - - void - addNodeCount(const SelectPruner &rhs); - - void - setInvalidVal(void); - - void - setInvalidConst(void); - - void - setTernaryConst(bool val); - - void - resolveTernaryConst(bool wantInverted); - - bool - isInvalidVal(void) const; - - bool - isNullVal(void) const; - - void - swap(SelectPruner &rhs); + void visitAndBranch(const document::select::And &expr) override; + void visitComparison(const document::select::Compare &expr) override; + void visitDocumentType(const document::select::DocType &expr) override; + void visitNotBranch(const document::select::Not &expr) override; + void visitOrBranch(const document::select::Or &expr) override; + void visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override; + void visitFunctionValueNode(const document::select::FunctionValueNode &expr) override; + void visitFieldValueNode(const document::select::FieldValueNode &expr) override; + void invertNode(); + const document::select::Operator &getOperator(const document::select::Operator &op); + void addNodeCount(const SelectPruner &rhs); + void setInvalidVal(); + void setInvalidConst(); + void setTernaryConst(bool val); + void resolveTernaryConst(bool wantInverted); + bool isInvalidVal() const; + bool isNullVal() const; + void swap(SelectPruner &rhs); }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp index 01f202fe1f4..a35eec12cbb 100644 --- a/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/state_reporter_utils.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.common.state_reporter_utils"); #include "state_reporter_utils.h" #include <vespa/vespalib/data/slime/cursor.h> @@ -28,4 +25,3 @@ StateReporterUtils::convertToSlime(const StatusReport &statusReport, } } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/common/subdbtype.h b/searchcore/src/vespa/searchcore/proton/common/subdbtype.h index 0864b0a8fe7..b782b46aa6b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/subdbtype.h +++ b/searchcore/src/vespa/searchcore/proton/common/subdbtype.h @@ -2,8 +2,7 @@ #pragma once -namespace proton -{ +namespace proton { /** * Enumeration of the different kinds of sub databases within a diff --git a/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh b/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh index 1b0b2440c45..4b2d8284aa7 100755 --- a/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh +++ b/searchcore/src/vespa/searchcore/proton/create-class-cpp.sh @@ -5,9 +5,6 @@ dir=`dirname $0` . "$dir/create-base.sh" cat <<EOF -#include <vespa/log/log.h> -LOG_SETUP(".$name"); -#include <vespa/fastos/fastos.h> #include "$name.h" $ns_open diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp index e5cc548e92c..4b73647a19c 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.cpp @@ -40,6 +40,7 @@ DocumentStoreExplorer::get_state(const Inserter &inserter, bool full) const object.setDouble("maxBucketSpread", storageStats.maxBucketSpread()); object.setLong("lastFlushedSerialNum", storageStats.lastFlushedSerialNum()); object.setLong("lastSerialNum", storageStats.lastSerialNum()); + object.setLong("docIdLimit", storageStats.docIdLimit()); setMemoryUsage(object, store.getMemoryUsage()); if (full) { const vespalib::string &baseDir = store.getBaseDir(); @@ -51,9 +52,9 @@ DocumentStoreExplorer::get_state(const Inserter &inserter, bool full) const chunkCursor.setLong("diskUsage", chunk.diskUsage()); chunkCursor.setLong("diskBloat", chunk.diskBloat()); chunkCursor.setDouble("bucketSpread", chunk.maxBucketSpread()); - chunkCursor.setLong("lastFlushedSerialNum", - chunk.lastFlushedSerialNum()); + chunkCursor.setLong("lastFlushedSerialNum", chunk.lastFlushedSerialNum()); chunkCursor.setLong("lastSerialNum", chunk.lastSerialNum()); + chunkCursor.setLong("docIdLimit", chunk.docIdLimit()); chunkCursor.setLong("nameid", chunk.nameId()); chunkCursor.setString("name", chunk.createName(baseDir)); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h index d885f18bfe5..9e822d06492 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h @@ -47,7 +47,7 @@ public: return _resultClass; } - uint32_t getNumDocs() const override { return _docStore.nextId(); } + uint32_t getNumDocs() const override { return _docStore.getDocIdLimit(); } search::docsummary::DocsumStoreValue getMappedDocsum(uint32_t docId) override; uint32_t getSummaryClassId() const override { return _resultClass->GetClassID(); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp index a38bee61480..d14ab4633c6 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "fieldcacherepo.h" + #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.fieldcacherepo"); -#include "fieldcacherepo.h" using namespace document; using namespace search::docsummary; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp index 0c332edba6a..59108a92727 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.docsummary.summarycompacttarget"); #include "summarycompacttarget.h" -#include <vespa/searchlib/util/dirtraverse.h> using search::IDocumentStore; using search::SerialNum; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp index f2007b2f988..f4587ef7287 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.docsummary.summaryflushtarget"); #include "summaryflushtarget.h" -#include <vespa/searchlib/util/dirtraverse.h> using search::IDocumentStore; using search::SerialNum; @@ -40,7 +36,7 @@ public: } virtual SerialNum - getFlushSerial(void) const override + getFlushSerial() const override { return _currSerial; } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h index 38fb0df3d52..238428e3084 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h @@ -6,7 +6,6 @@ namespace proton { - /** * This class implements the IFlushTarget interface to proxy a summary manager. */ diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index a3282e9e3ec..0253e943883 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -4,11 +4,13 @@ #include "summarycompacttarget.h" #include "summaryflushtarget.h" #include "summarymanager.h" -#include <vespa/searchlib/docstore/logdocumentstore.h> -#include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/config/print/ostreamconfigwriter.h> #include <vespa/juniper/rpinterface.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> +#include <vespa/searchlib/docstore/logdocumentstore.h> +#include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.summarymanager"); @@ -22,6 +24,7 @@ using namespace vespa::config::search; using vespalib::make_string; using vespalib::IllegalArgumentException; using search::DocumentStore; +using search::IDocumentStore; using search::LogDocumentStore; using search::LogDataStore; using search::WriteableFileChunk; @@ -171,13 +174,29 @@ SummaryManager::removeDocument(uint64_t syncToken, search::DocumentIdT lid) _currentSerial = syncToken; } +namespace { + +IFlushTarget::SP +createShrinkLidSpaceFlushTarget(IDocumentStore::SP docStore) +{ + return std::make_shared<ShrinkLidSpaceFlushTarget>("summary.shrink", + IFlushTarget::Type::GC, + IFlushTarget::Component::DOCUMENT_STORE, + docStore->lastSyncToken(), + docStore->getLastFlushTime(), + docStore); +} + +} + IFlushTarget::List SummaryManager::getFlushTargets() { IFlushTarget::List ret; - ret.push_back(IFlushTarget::SP(new SummaryFlushTarget(getBackingStore()))); + ret.push_back(std::make_shared<SummaryFlushTarget>(getBackingStore())); if (dynamic_cast<LogDocumentStore *>(_docStore.get()) != NULL) { - ret.push_back(IFlushTarget::SP(new SummaryCompactTarget(getBackingStore()))); + ret.push_back(std::make_shared<SummaryCompactTarget>(getBackingStore())); } + ret.push_back(createShrinkLidSpaceFlushTarget(_docStore)); return ret; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp index bcd9302cde6..1d91e1e812a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.document_meta_store_explorer"); #include "document_meta_store_explorer.h" - #include <vespa/vespalib/data/slime/cursor.h> using vespalib::slime::Cursor; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp index 20381e51412..5dc3308436d 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_meta_store_initializer_result.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 209d2cf07f3..742764f7c64 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -95,12 +95,12 @@ public: } uint8_t - getNextBucketUsedBits(void) { + getNextBucketUsedBits() { return _bucketUsedBitsReader.readHostOrder(); } Timestamp - getNextTimestamp(void) { + getNextTimestamp() { return _timestampReader.readHostOrder(); } @@ -988,7 +988,7 @@ DocumentMetaStore::compactLidSpace(uint32_t wantedLidLimit) } void -DocumentMetaStore::holdUnblockShrinkLidSpace(void) +DocumentMetaStore::holdUnblockShrinkLidSpace() { assert(_shrinkLidSpaceBlockers > 0); GenerationHeldBase::UP hold(new ShrinkBlockHeld(*this)); @@ -997,14 +997,14 @@ DocumentMetaStore::holdUnblockShrinkLidSpace(void) } void -DocumentMetaStore::unblockShrinkLidSpace(void) +DocumentMetaStore::unblockShrinkLidSpace() { assert(_shrinkLidSpaceBlockers > 0); --_shrinkLidSpaceBlockers; } bool -DocumentMetaStore::canShrinkLidSpace(void) const +DocumentMetaStore::canShrinkLidSpace() const { return AttributeVector::canShrinkLidSpace() && _shrinkLidSpaceBlockers == 0; @@ -1019,6 +1019,21 @@ DocumentMetaStore::onShrinkLidSpace() setNumDocs(committedDocIdLimit); } +size_t +DocumentMetaStore::getEstimatedShrinkLidSpaceGain() const +{ + size_t canFree = 0; + if (canShrinkLidSpace()) { + uint32_t committedDocIdLimit = getCommittedDocIdLimit(); + uint32_t numDocs = getNumDocs(); + if (committedDocIdLimit < numDocs) { + canFree = sizeof(RawDocumentMetaData) * + (numDocs - committedDocIdLimit); + } + } + return canFree; +} + BucketId DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint32_t lid) const { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 200535c7ae5..1f1f933700f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -274,6 +274,7 @@ public: */ void unblockShrinkLidSpace(); void onShrinkLidSpace() override; + virtual size_t getEstimatedShrinkLidSpaceGain() const override; uint64_t getEstimatedSaveByteSize() const override; virtual uint32_t getVersion() const override; void setTrackDocumentSizes(bool trackDocumentSizes) { _trackDocumentSizes = trackDocumentSizes; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp index 68b33438642..26f63b6f121 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp @@ -31,7 +31,7 @@ DocumentMetaStoreContext::DocumentMetaStoreContext(const search::AttributeVector DocumentMetaStoreContext::~DocumentMetaStoreContext() {} void -DocumentMetaStoreContext::constructFreeList(void) +DocumentMetaStoreContext::constructFreeList() { _metaStore->constructFreeList(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 33c90819e33..41f038d0e93 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -1,25 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.documentmetastoreflushtarget"); -#include <vespa/searchcore/proton/attribute/attributedisklayout.h> #include "documentmetastoreflushtarget.h" +#include "documentmetastore.h" +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/server/itlssyncer.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchlib/attribute/attributefilesavetarget.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributesaver.h> #include <vespa/searchlib/util/dirtraverse.h> +#include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/closuretask.h> -#include <vespa/searchcore/proton/common/hw_info.h> #include <fstream> -#include <vespa/searchlib/common/serialnumfileheadercontext.h> -#include <vespa/searchcore/proton/server/itlssyncer.h> -#include <vespa/searchcore/proton/attribute/attributedisklayout.h> -#include <vespa/searchcore/proton/attribute/attribute_directory.h> -#include "documentmetastore.h" + +#include <vespa/log/log.h> +LOG_SETUP(".proton.documentmetastore.documentmetastoreflushtarget"); using namespace search; using namespace vespalib; @@ -55,7 +52,7 @@ public: virtual void run() override; virtual SerialNum - getFlushSerial(void) const override + getFlushSerial() const override { return _syncToken; } @@ -71,9 +68,6 @@ Flusher(DocumentMetaStoreFlushTarget &dmsft, { DocumentMetaStore &dms = *_dmsft._dms; // Called by document db executor - if (dms.canShrinkLidSpace()) { - dms.shrinkLidSpace(); - } _flushDir = writer.getSnapshotDir(syncToken); vespalib::string newBaseFileName(_flushDir + "/" + dms.getName()); dms.setBaseFileName(newBaseFileName); @@ -176,7 +170,7 @@ DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms, const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, const HwInfo &hwInfo) - : IFlushTarget("documentmetastore", Type::SYNC, Component::ATTRIBUTE), + : IFlushTarget("documentmetastore.flush", Type::SYNC, Component::ATTRIBUTE), _dms(dms), _tlsSyncer(tlsSyncer), _baseDir(baseDir), @@ -209,18 +203,7 @@ IFlushTarget::MemoryGain DocumentMetaStoreFlushTarget::getApproxMemoryGain() const { int64_t used(_dms->getStatus().getUsed()); - int64_t canFree = 0; - if (_dms->canShrinkLidSpace()) { - uint32_t committedDocIdLimit = _dms->getCommittedDocIdLimit(); - uint32_t numDocs = _dms->getNumDocs(); - if (committedDocIdLimit < numDocs) { - canFree = sizeof(RawDocumentMetaData) * - (numDocs - committedDocIdLimit); - if (canFree > used) - canFree = used; - } - } - return MemoryGain(used, used - canFree); + return MemoryGain(used, used); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h index 598744978da..307eb43ab12 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h @@ -2,11 +2,9 @@ #pragma once -namespace proton -{ - -namespace documentmetastore -{ +#include <cstdint> +#include <vector> +namespace proton::documentmetastore { /** * Interface used to delay reuse of lids until references to the lids have @@ -39,5 +37,3 @@ public: }; } - -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp index 55f8f3f7cbe..642fc1b1154 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_gid_key_comparator.cpp @@ -1,13 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.lid_gid_key_comparator"); - #include "lid_gid_key_comparator.h" -namespace proton { -namespace documentmetastore { +namespace proton::documentmetastore { const search::IDocumentMetaStore::DocId LidGidKeyComparator::FIND_DOC_ID = std::numeric_limits<DocId>::max(); @@ -31,4 +26,3 @@ LidGidKeyComparator::LidGidKeyComparator(const RawDocumentMetaData &metaData, } } -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index 84c579ba4e6..7a618d34250 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -1,15 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "lid_reuse_delayer_config.h" #include <vespa/searchcore/proton/server/documentdbconfig.h> -namespace proton -{ - -namespace documentmetastore -{ - +namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig() : _visibilityDelay(0), @@ -29,4 +23,3 @@ LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & } // namespace proton::documentmetastore -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index 18d2f55541c..975b5c268e9 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -2,13 +2,12 @@ #pragma once -namespace proton -{ +#include <vespa/fastos/timestamp.h> +namespace proton { class DocumentDBConfig; -namespace documentmetastore -{ +namespace documentmetastore { /* * Class representing configuration for lid reuse delayer. diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index 02dcdc2fb03..06578107ee6 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -1,18 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.lidreusedelayer"); +#include "lidreusedelayer.h" +#include "i_store.h" #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/closuretask.h> -#include "i_store.h" -#include "lidreusedelayer.h" -namespace proton -{ - -namespace documentmetastore -{ +namespace proton::documentmetastore { using searchcorespi::index::IThreadingService; using vespalib::makeClosure; @@ -106,4 +99,3 @@ LidReuseDelayer::getReuseLids() } -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index 8d3264ba241..8b7a677accd 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -1,12 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.documentmetastore.lidstatevector"); #include "lidstatevector.h" -namespace proton -{ +namespace proton { using vespalib::GenerationHolder; @@ -68,7 +64,7 @@ LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity) void -LidStateVector::updateLowest(void) +LidStateVector::updateLowest() { if (_lowest >= _bv.size()) return; @@ -81,7 +77,7 @@ LidStateVector::updateLowest(void) void -LidStateVector::updateHighest(void) +LidStateVector::updateHighest() { if (_highest == 0) return; @@ -124,28 +120,28 @@ LidStateVector::clearBit(unsigned int idx) bool -LidStateVector::empty(void) const +LidStateVector::empty() const { return _count == 0u; } unsigned int -LidStateVector::getLowest(void) const +LidStateVector::getLowest() const { return _lowest; } unsigned int -LidStateVector::getHighest(void) const +LidStateVector::getHighest() const { return _highest; } uint32_t -LidStateVector::internalCount(void) +LidStateVector::internalCount() { // Called by document db executor thread. return _bv.countTrueBits(); @@ -153,7 +149,7 @@ LidStateVector::internalCount(void) uint32_t -LidStateVector::count(void) const +LidStateVector::count() const { // Called by document db executor thread or metrics related threads return _count; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h index 8c8c41f6184..c870fe0d953 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h @@ -17,20 +17,20 @@ class LidStateVector bool _trackHighest; void - updateLowest(void); + updateLowest(); void - updateHighest(void); + updateHighest(); inline void - maybeUpdateLowest(void) + maybeUpdateLowest() { if (_trackLowest && _lowest < _bv.size() && !_bv.testBit(_lowest)) updateLowest(); } inline void - maybeUpdateHighest(void) + maybeUpdateHighest() { if (_trackHighest && _highest != 0 && !_bv.testBit(_highest)) updateHighest(); @@ -41,7 +41,7 @@ class LidStateVector * write thread. */ uint32_t - internalCount(void); + internalCount(); public: LidStateVector(unsigned int newSize, @@ -68,32 +68,32 @@ public: } inline unsigned int - size(void) const + size() const { return _bv.size(); } inline unsigned int - byteSize(void) const + byteSize() const { return _bv.extraByteSize() + sizeof(LidStateVector); } bool - empty(void) const; + empty() const; unsigned int - getLowest(void) const; + getLowest() const; unsigned int - getHighest(void) const; + getHighest() const; /** * Get cached number of bits set in vector. Called by read or * write thread. Write thread must updated cached number as needed. */ uint32_t - count(void) const; + count() const; unsigned int getNextTrueBit(unsigned int idx) const diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h index 29fa597b3ec..68a354c9984 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h @@ -23,7 +23,7 @@ struct RawDocumentMetaData uint16_t _docSizeHigh; Timestamp _timestamp; - RawDocumentMetaData(void) + RawDocumentMetaData() : _gid(), _bucketUsedBits(BucketId::minNumBits), _docSizeLow(0), @@ -80,7 +80,7 @@ struct RawDocumentMetaData _bucketUsedBits = bucketUsedBits; } - Timestamp getTimestamp(void) const { return _timestamp; } + Timestamp getTimestamp() const { return _timestamp; } void setTimestamp(const Timestamp ×tamp) { _timestamp = timestamp; } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp index 047813d8a05..e4211fe14cc 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/compact_lid_space_operation.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.compact_lid_space_operation"); - #include "compact_lid_space_operation.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp index d761677542e..cb696c97bac 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.createbucketoperation"); #include "createbucketoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using document::BucketId; using document::DocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp index 964f2e44702..5fa5132d41d 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.deletebucketoperation"); #include "deletebucketoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using document::BucketId; using document::DocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index f4ff9255547..b9df5160f14 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "documentoperation.h" -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/document/base/documentid.h> +#include <cassert> using document::BucketId; using document::DocumentId; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index 4b22c868628..28cff31d624 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -31,19 +31,19 @@ protected: public: virtual - ~DocumentOperation(void) + ~DocumentOperation() { } const document::BucketId & - getBucketId(void) const + getBucketId() const { return _bucketId; } storage::spi::Timestamp - getTimestamp(void) const + getTimestamp() const { return _timestamp; } @@ -61,19 +61,19 @@ public: } uint32_t - getSubDbId(void) const + getSubDbId() const { return _dbdId.getSubDbId(); } uint32_t - getPrevSubDbId(void) const + getPrevSubDbId() const { return _prevDbdId.getSubDbId(); } bool - getValidDbdId(void) const + getValidDbdId() const { return _dbdId.valid(); } @@ -85,7 +85,7 @@ public: } bool - getValidPrevDbdId(void) const + getValidPrevDbdId() const { return _prevDbdId.valid(); } @@ -97,12 +97,12 @@ public: } bool - changedDbdId(void) const + changedDbdId() const { return _dbdId != _prevDbdId; } bool - getPrevMarkedAsRemoved(void) const + getPrevMarkedAsRemoved() const { return _prevMarkedAsRemoved; } @@ -114,13 +114,13 @@ public: } DbDocumentId - getDbDocumentId(void) const + getDbDocumentId() const { return _dbdId; } DbDocumentId - getPrevDbDocumentId(void) const + getPrevDbDocumentId() const { return _prevDbdId; } @@ -148,13 +148,13 @@ public: } bool - getValidNewOrPrevDbdId(void) const + getValidNewOrPrevDbdId() const { return getValidDbdId() || getValidPrevDbdId(); } bool - notMovingLidInSameSubDb(void) const + notMovingLidInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || @@ -163,7 +163,7 @@ public: } bool - movingLidIfInSameSubDb(void) const + movingLidIfInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || @@ -172,7 +172,7 @@ public: } storage::spi::Timestamp - getPrevTimestamp(void) const + getPrevTimestamp() const { return _prevTimestamp; } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp index 789d3ec97ee..cee360323af 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.feedoperation"); #include "feedoperation.h" diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp index afd96720baf..8e51c4e5e94 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/joinbucketsoperation.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.joinbucketsoperation"); #include "joinbucketsoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using document::BucketId; using document::DocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp index 387b41bc010..b8b32f31105 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.lidvectorcontext"); #include "lidvectorcontext.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.feedoperation.lidvectorcontext"); + using search::BitVector; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp index 59c8b2dda33..2cbfee1d5c9 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "moveoperation.h" -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/document/fieldvalue/document.h> +#include <cassert> using document::BucketId; using document::Document; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp index 2044fdea43f..33e25786651 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/newconfigoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.newconfigoperation"); #include "newconfigoperation.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp index 0ea7977f2bc..1330fd6eda9 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/noopoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.noopoperation"); #include "noopoperation.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index b257fb14d57..1a18fee265f 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -1,10 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation"); #include "pruneremoveddocumentsoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation"); using document::DocumentTypeRepo; using search::DocumentIdT; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp index 701efc368ad..41fd2b1482b 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "putoperation.h" #include <vespa/document/fieldvalue/document.h> diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp index 247a8dfc803..e11677db6fe 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.removedocumentsoperation"); #include "removedocumentsoperation.h" diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp index 58d802716b0..446362dbbab 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.removeoperation"); #include "removeoperation.h" diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp index 3fbf0e93fdf..a188ae866e6 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/splitbucketoperation.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.splitbucketoperation"); #include "splitbucketoperation.h" #include <vespa/vespalib/util/stringfmt.h> +#include <cassert> using document::BucketId; using document::DocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp index 9413e5c0dbc..10de16cadfd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/spoolerreplayoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.spoolerreplayoperation"); #include "spoolerreplayoperation.h" #include <vespa/vespalib/util/stringfmt.h> @@ -31,7 +28,6 @@ SpoolerReplayOperation::SpoolerReplayOperation(Type type, void SpoolerReplayOperation::serialize(vespalib::nbostream &os) const { - LOG(debug, "serialize(): %s", toString().c_str()); os << _spoolerSerialNum; } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index e69b64548bc..c5225ca8f20 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "updateoperation.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/document/base/exceptions.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.feedoperation.updateoperation"); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp index 5e4f2046481..8e0dd06cb1b 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/wipehistoryoperation.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedoperation.wipehistoryoperation"); #include "wipehistoryoperation.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index dddcfbd9327..bd7bc949e8f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(searchcore_flushengine STATIC SOURCES cachedflushtarget.cpp + shrink_lid_space_flush_target.cpp flush_all_strategy.cpp flushcontext.cpp flushengine.cpp diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp index fe10efb816e..81c8a11426a 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.cachedflushtarget"); #include "cachedflushtarget.h" diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp index 8216c9b1391..a763be5e111 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.flush_engine_explorer"); #include "flush_engine_explorer.h" #include <vespa/vespalib/data/slime/cursor.h> diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp index e83b0adc3f5..61f025c8e53 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.flush_target_candidates"); #include "flush_target_candidates.h" #include "tls_stats.h" diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp index 1624bc3b51c..f5ccfc7e0cb 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.flushcontext"); #include "flushcontext.h" +#include <vespa/log/log.h> +LOG_SETUP(".proton.flushengine.flushcontext"); + namespace proton { FlushContext::FlushContext( diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index e75b44a2235..f036d12145d 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -118,7 +118,7 @@ FlushEngine::triggerFlush() } void -FlushEngine::kick(void) +FlushEngine::kick() { MonitorGuard guard(_monitor); LOG(debug, "Kicking flush engine"); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index d4ebb687e87..f89bf15db80 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -130,7 +130,7 @@ public: */ void triggerFlush(); - void kick(void); + void kick(); /** * Registers a new flush handler for the given document type. If another diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp index 92133c6d656..cc1e6b06190 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.flushtargetproxy"); #include "flushtargetproxy.h" diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp index 8af59d3079f..b04f2ae6407 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.flushtask"); #include "flushtask.h" @@ -14,7 +11,7 @@ FlushTask::FlushTask(uint32_t taskId, _engine(engine), _context(ctx) { - LOG_ASSERT(_context.get() != NULL); + assert(_context.get() != NULL); } FlushTask::~FlushTask() diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp index 1bf45bd332e..2244759e866 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.prepare_restart_flush_strategy"); #include "prepare_restart_flush_strategy.h" #include "flush_target_candidates.h" #include "tls_stats_map.h" +#include <sstream> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.flushengine.prepare_restart_flush_strategy"); namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp new file mode 100644 index 00000000000..1341410a39d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "shrink_lid_space_flush_target.h" +#include <vespa/searchlib/common/i_compactable_lid_space.h> + +namespace proton { + +using searchcorespi::IFlushTarget; +using searchcorespi::FlushStats; +using searchcorespi::FlushTask; + +class ShrinkLidSpaceFlushTarget::Flusher : public FlushTask +{ + ShrinkLidSpaceFlushTarget &_target; + SerialNum _flushSerialNum; +public: + Flusher(ShrinkLidSpaceFlushTarget &target, SerialNum flushSerialNum); + virtual void run() override; + virtual search::SerialNum getFlushSerial() const override; +}; + +ShrinkLidSpaceFlushTarget::Flusher::Flusher(ShrinkLidSpaceFlushTarget &target, SerialNum flushSerialNum) + : FlushTask(), + _target(target), + _flushSerialNum(flushSerialNum) +{ + _target._target->shrinkLidSpace(); +} + +void +ShrinkLidSpaceFlushTarget::Flusher::run() +{ + _target._flushedSerialNum = _flushSerialNum; + _target._lastFlushTime = fastos::ClockSystem::now(); +} + +search::SerialNum +ShrinkLidSpaceFlushTarget::Flusher::getFlushSerial() const +{ + return _flushSerialNum; +} + +ShrinkLidSpaceFlushTarget::ShrinkLidSpaceFlushTarget(const vespalib::string &name, + Type type, + Component component, + SerialNum flushedSerialNum, + Time lastFlushTime, + std::shared_ptr<ICompactableLidSpace> target) + : IFlushTarget(name, type, component), + + _target(std::move(target)), + _flushedSerialNum(flushedSerialNum), + _lastFlushTime(lastFlushTime), + _lastStats() +{ +} + +IFlushTarget::MemoryGain +ShrinkLidSpaceFlushTarget::getApproxMemoryGain() const +{ + int64_t canFree = _target->getEstimatedShrinkLidSpaceGain(); + return MemoryGain(canFree, 0); +} + +IFlushTarget::DiskGain +ShrinkLidSpaceFlushTarget::getApproxDiskGain() const +{ + return DiskGain(0, 0); +} + +IFlushTarget::SerialNum +ShrinkLidSpaceFlushTarget::getFlushedSerialNum() const +{ + return _flushedSerialNum; +} + +IFlushTarget::Time +ShrinkLidSpaceFlushTarget::getLastFlushTime() const +{ + return _lastFlushTime; +} + +bool +ShrinkLidSpaceFlushTarget::needUrgentFlush() const +{ + return false; +} + +IFlushTarget::Task::UP +ShrinkLidSpaceFlushTarget::initFlush(SerialNum currentSerial) +{ + if (currentSerial < _flushedSerialNum) { + _lastFlushTime = fastos::ClockSystem::now(); + return IFlushTarget::Task::UP(); + } else if (!_target->canShrinkLidSpace()) { + _flushedSerialNum = currentSerial; + _lastFlushTime = fastos::ClockSystem::now(); + return IFlushTarget::Task::UP(); + } else { + return std::make_unique<Flusher>(*this, currentSerial); + } +} + +FlushStats +ShrinkLidSpaceFlushTarget::getLastFlushStats() const +{ + return _lastStats; +} + +uint64_t +ShrinkLidSpaceFlushTarget::getApproxBytesToWriteToDisk() const +{ + return 0; +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h new file mode 100644 index 00000000000..18bda6bda4e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcorespi/flush/iflushtarget.h> + +namespace search::common { class ICompactableLidSpace; } + +namespace proton { + + +/** + * Implements a flush target that shrinks lid space in target. + */ +class ShrinkLidSpaceFlushTarget : public searchcorespi::IFlushTarget +{ + /** + * Task representing that shrinking has been performed. + **/ + class Flusher; + using ICompactableLidSpace = search::common::ICompactableLidSpace; + using FlushStats = searchcorespi::FlushStats; + std::shared_ptr<ICompactableLidSpace> _target; + SerialNum _flushedSerialNum; + Time _lastFlushTime; + FlushStats _lastStats; + +public: + /** + * Constructs a new instance of this class. + * + * @param name The handler-wide unique name of this target. + * @param type The flush type of this target. + * @param component The component type of this target. + * @param flushedSerialNum When target shrank lid space last time + * @param target The target supporting lid space compaction + */ + ShrinkLidSpaceFlushTarget(const vespalib::string &name, + Type type, + Component component, + SerialNum flushedSerialNum, + Time lastFlushTime, + std::shared_ptr<ICompactableLidSpace> target); + + // Implements IFlushTarget. + virtual MemoryGain getApproxMemoryGain() const override; + virtual DiskGain getApproxDiskGain() const override; + virtual SerialNum getFlushedSerialNum() const override; + virtual Time getLastFlushTime() const override; + virtual bool needUrgentFlush() const override; + virtual Task::UP initFlush(SerialNum currentSerial) override; + virtual searchcorespi::FlushStats getLastFlushStats() const override; + virtual uint64_t getApproxBytesToWriteToDisk() const override; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp index 49d400c95f7..5e2065a599a 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.flushengine.threadedflushtarget"); #include "threadedflushtarget.h" -#include <vespa/vespalib/util/executor.h> #include <vespa/searchcore/proton/server/igetserialnum.h> #include <vespa/searchlib/common/lambdatask.h> #include <future> +#include <cassert> using searchcorespi::IFlushTarget; using searchcorespi::FlushStats; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h index 07e726d2032..963d805ad2c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -namespace proton { -namespace flushengine { +#include <cstdint> + +namespace proton::flushengine { /* * Class representing statistics for a transaction log server domain used to @@ -37,5 +38,4 @@ public: uint64_t getLastSerial() const { return _lastSerial; } }; -} // namespace proton::flushengine -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp index 6819849e959..06fdc46d71b 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tls_stats_factory.h" #include "tls_stats_map.h" #include <vespa/searchlib/transactionlog/translogserver.h> diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h index d580d58b964..85c20413f42 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/tls_stats_factory.h @@ -2,10 +2,11 @@ #pragma once #include "i_tls_stats_factory.h" +#include <memory> namespace search { namespace transactionlog { class TransLogServer; } } -namespace proton { -namespace flushengine { + +namespace proton::flushengine { /* * Class used to create statistics for a transaction log server over @@ -21,5 +22,4 @@ public: virtual TlsStatsMap create() override; }; -} // namespace proton::flushengine -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp index ad78835fa37..e5730b66edb 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.index.diskindexwrapper"); #include "diskindexwrapper.h" #include <vespa/searchcorespi/index/indexreadutilities.h> diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp index 4f882660855..18578be184b 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.index.indexmanagerinitializer"); #include "index_manager_initializer.h" #include <vespa/vespalib/io/fileutil.h> -namespace proton -{ +#include <vespa/log/log.h> +LOG_SETUP(".proton.index.indexmanagerinitializer"); + +namespace proton { IndexManagerInitializer:: IndexManagerInitializer(const vespalib::string &baseDir, diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 5ba6e11331c..288dd1067c6 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.index.indexmanager"); #include "indexmanager.h" #include "diskindexwrapper.h" diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp index 73a86c805e2..cb687b80a08 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.index.memoryindexwrapper"); #include "memoryindexwrapper.h" #include <vespa/searchlib/common/serialnumfileheadercontext.h> diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp index 041b4546e70..09b01180432 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "initializer_task.h" -namespace proton { - -namespace initializer { +namespace proton::initializer { InitializerTask::InitializerTask() : _state(State::BLOCKED), @@ -25,7 +22,5 @@ InitializerTask::addDependency(SP dependency) _dependencies.emplace_back(std::move(dependency)); } - } // namespace proton::initializer -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index e9c0cba6069..78fd38ea87a 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "matchengine.h" #include <vespa/searchcore/proton/common/state_reporter_utils.h> #include <vespa/vespalib/data/slime/cursor.h> diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 8574770352c..999c997d221 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -144,7 +144,7 @@ Query::optimize() } void -Query::fetchPostings(void) +Query::fetchPostings() { _blueprint->fetchPostings(true); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index aa9ee52d82c..484d961870b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -84,7 +84,7 @@ public: * test to verify the original query without optimization. **/ void optimize(); - void fetchPostings(void); + void fetchPostings(); void freeze(); /** diff --git a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp index e6a03c8df52..a0a63cadbdc 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_metrics.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp index ce6152fe464..c4564567d86 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp @@ -48,6 +48,8 @@ DocumentDBJobTrackers::trackFlushTargets(const IFlushTarget::List &flushTargets) for (const auto &ft : flushTargets) { if (ft->getComponent() == FTC::ATTRIBUTE && ft->getType() == FTT::SYNC) { retval.push_back(trackFlushTarget(_attributeFlush, ft)); + } else if (ft->getComponent() == FTC::ATTRIBUTE && ft->getType() == FTT::GC) { + retval.push_back(trackFlushTarget(_attributeFlush, ft)); } else if (ft->getComponent() == FTC::INDEX && ft->getType() == FTT::FLUSH) { retval.push_back(trackFlushTarget(_memoryIndexFlush, ft)); } else if (ft->getComponent() == FTC::INDEX && ft->getType() == FTT::GC) { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp index 9f60951393c..494f54add46 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/feed_metrics.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.feedengine.feedmetrics"); #include "feed_metrics.h" -#include <vespa/vespalib/util/sync.h> using vespalib::LockGuard; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp index 95fb6a0993d..05a8bf405cc 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_load_sampler.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.metrics.job_load_sampler"); #include "job_load_sampler.h" +#include <cassert> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp index a96164d506c..7c600d1fe8c 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_task.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.metrics.job_tracked_flush_task"); #include "job_tracked_flush_task.h" using searchcorespi::FlushTask; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp index 5c7dd5d8749..c6ea2970135 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracker.cpp @@ -1,14 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.metrics.job_tracker"); #include "job_tracker.h" -#include <vespa/fastos/timestamp.h> -#include <chrono> - -using fastos::TimeStamp; -using fastos::ClockSystem; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp index 10083daa769..0591770bdef 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "legacy_attribute_metrics.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index 125ffde7530..4586bd51ae2 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "metrics_engine.h" +#include "attribute_metrics_collection.h" #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/metricmanager.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.metricsengine"); -#include "attribute_metrics_collection.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 9df67a6bf11..79629410b00 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -4,6 +4,7 @@ #include <vespa/document/select/gid_filter.h> #include <vespa/document/select/node.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/log/log.h> LOG_SETUP(".proton.persistenceengine.document_iterator"); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp index 81a56f4aac6..a7bc34a2130 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/persistence/spi/read_consistency.h> -#include <vespa/fastos/fastos.h> #include "i_document_retriever.h" +#include <vespa/persistence/spi/read_consistency.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h index b8952edb990..ef3000dc1f7 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h @@ -11,7 +11,7 @@ class IPersistenceEngineOwner { public: virtual - ~IPersistenceEngineOwner(void) + ~IPersistenceEngineOwner() { } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 2a17cb97952..734e3e2da55 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -656,7 +656,7 @@ PersistenceEngine::maintain(const Bucket& , MaintenanceLevel) } void -PersistenceEngine::destroyIterators(void) +PersistenceEngine::destroyIterators() { Context context(storage::spi::LoadType(0, "default"), storage::spi::Priority(0x80), @@ -689,7 +689,7 @@ PersistenceEngine::saveClusterState(const ClusterState &calc) } PersistenceEngine::ClusterState::SP -PersistenceEngine::savedClusterState(void) const +PersistenceEngine::savedClusterState() const { LockGuard guard(_lock); return _clusterState; @@ -738,7 +738,7 @@ public: } } - const BucketIdMap & getBucketMap(void) const { return _bucketMap; } + const BucketIdMap & getBucketMap() const { return _bucketMap; } }; void @@ -773,7 +773,7 @@ PersistenceEngine::populateInitialBucketDB(IPersistenceHandler &targetHandler) } std::unique_lock<std::shared_timed_mutex> -PersistenceEngine::getWLock(void) const +PersistenceEngine::getWLock() const { return std::unique_lock<std::shared_timed_mutex>(_rwMutex); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index 40d59453673..c3cda712385 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -92,7 +92,7 @@ private: HandlerSnapshot::UP getHandlerSnapshot(const document::DocumentId &) const; void saveClusterState(const ClusterState &calc); - ClusterState::SP savedClusterState(void) const; + ClusterState::SP savedClusterState() const; public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -134,7 +134,7 @@ public: void propagateSavedClusterState(IPersistenceHandler &handler); void grabExtraModifiedBuckets(IPersistenceHandler &handler); void populateInitialBucketDB(IPersistenceHandler &targetHandler); - std::unique_lock<std::shared_timed_mutex> getWLock(void) const; + std::unique_lock<std::shared_timed_mutex> getWLock() const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp index 78eb1cfb3f6..94bac320ef3 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transport_latch.h" +#include <vespa/vespalib/util/stringfmt.h> using storage::spi::Result; diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference.cpp index b9306a17bfb..3c686c10675 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference.cpp @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_db_reference.h" -#include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/searchlib/attribute/iattributemanager.h> #include "gid_to_lid_mapper_factory.h" -#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include "gid_to_lid_change_registrator.h" +#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> +#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/iattributemanager.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp index 80f5d160d8c..16a16323c9f 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_registry.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_db_reference_registry.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp index 58cd50fdd46..c60ee3f176c 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp @@ -1,20 +1,19 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_db_reference_resolver.h" #include "gid_to_lid_change_listener.h" #include "gid_to_lid_change_registrator.h" #include "i_document_db_reference.h" #include "i_document_db_reference_registry.h" -#include <vespa/config-imported-fields.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/datatype/referencedatatype.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> +#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/config-imported-fields.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/referencedatatype.h> +#include <vespa/document/repo/documenttyperepo.h> using document::DataType; using document::DocumentType; diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp index 8b5f0fb5515..1a6d58592ff 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "gid_to_lid_mapper.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp index 7582b7d304a..f9902589cee 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_mapper_factory.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "gid_to_lid_mapper_factory.h" #include "gid_to_lid_mapper.h" #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h index 9130f0072a6..2de5928cdf9 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_document_db_reference_resolver.h @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <memory> + namespace search { class IAttributeManager; } namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index c5c46b7f1af..70251012554 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute_reprocessing_initializer.h" #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h index 5f695fa87f1..5a3b4af1b48 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_task.h @@ -2,6 +2,9 @@ #pragma once +#include <memory> +#include <vector> + namespace proton { /** @@ -23,31 +26,22 @@ public: Progress() : _progress(0.0), _weight(0.0) - { - } + {} Progress(double progress, double weight) : _progress(progress), _weight(weight) - { - } + {} }; - virtual - ~IReprocessingTask() - { - } + virtual ~IReprocessingTask() {} /** * Run reprocessing task. */ - virtual void - run() = 0; + virtual void run() = 0; - virtual Progress - getProgress() const = 0; + virtual Progress getProgress() const = 0; }; - } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp index 59b3966454a..62eee35df5f 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "reprocess_documents_task.h" #include "attribute_reprocessing_initializer.h" #include "document_reprocessing_handler.h" #include <vespa/searchcore/proton/common/eventlogger.h> -namespace proton -{ - +namespace proton { ReprocessDocumentsTask:: ReprocessDocumentsTask(IReprocessingInitializer &initializer, diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp index 4891829bbea..52e39176663 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <mutex> #include "reprocessingrunner.h" #include "i_reprocessing_task.h" +#include <mutex> -namespace proton -{ - +namespace proton { ReprocessingRunner::ReprocessingRunner() : _lock(), @@ -85,5 +82,4 @@ ReprocessingRunner::getProgress() const return weightedProgress / weight; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h index e9f4a3f31b7..41bef8815bd 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocessingrunner.h @@ -3,9 +3,10 @@ #pragma once #include <mutex> +#include <memory> +#include <vector> -namespace proton -{ +namespace proton { class IReprocessingTask; @@ -29,22 +30,11 @@ private: public: ReprocessingRunner(); - void - addTasks(const ReprocessingTasks &tasks); - - void - run(); - - void - reset(); - - bool - empty() const; - - double - getProgress() const; + void addTasks(const ReprocessingTasks &tasks); + void run(); + void reset(); + bool empty() const; + double getProgress() const; }; - } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 18c000e5f4e..8333e180a0a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_library(searchcore_server STATIC bucketmovejob.cpp clusterstatehandler.cpp combiningfeedview.cpp - data_directory_upgrader.cpp ddbstate.cpp disk_mem_usage_filter.cpp disk_mem_usage_sampler.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index ca20d65f34b..5599b7be360 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -51,7 +51,7 @@ BootstrapConfig::operator==(const BootstrapConfig &rhs) const bool -BootstrapConfig::valid(void) const +BootstrapConfig::valid() const { return _documenttypes.get() != NULL && _repo.get() != NULL && diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 0f83ae379b3..749aa5a48fb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -62,19 +62,19 @@ public: } const vespa::config::search::core::ProtonConfig & - getProtonConfig(void) const + getProtonConfig() const { return *_proton; } const ProtonConfigSP & - getProtonConfigSP(void) const + getProtonConfigSP() const { return _proton; } const search::TuneFileDocumentDB::SP & - getTuneFileDocumentDBSP(void) const + getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } @@ -98,7 +98,7 @@ public: operator==(const BootstrapConfig &rhs) const; bool - valid(void) const; + valid() const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index d98912ffec1..f51d3df2476 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.buckethandler"); #include "buckethandler.h" -#include <vespa/vespalib/util/closuretask.h> #include "ibucketstatechangedhandler.h" +#include <vespa/vespalib/util/closuretask.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.buckethandler"); using document::BucketId; using storage::spi::Bucket; diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h index d1188c022f7..efdb7b8b073 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h @@ -38,7 +38,7 @@ private: * up to down in cluster state. Called by document db executor thread. */ void - deactivateAllActiveBuckets(void); + deactivateAllActiveBuckets(); public: /** diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index d5da1cd1868..a95079cca0a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.bucketmovejob"); #include "bucketmovejob.h" #include "imaintenancejobrunner.h" #include "ibucketstatechangednotifier.h" @@ -11,6 +8,9 @@ LOG_SETUP(".proton.server.bucketmovejob"); #include "i_disk_mem_usage_notifier.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.bucketmovejob"); + using document::BucketId; using storage::spi::BucketInfo; diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index 7472a321437..7b432588b7a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -70,7 +70,7 @@ public: bool hasReadyBucketDocs() const { return _itr->second.getReadyCount() != 0; } bool hasNotReadyBucketDocs() const { return _itr->second.getNotReadyCount() != 0; } - ScanIterator & operator++(void) { + ScanIterator & operator++() { ++_itr; return *this; } diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp index bca74b10058..60f35934831 100644 --- a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.clusterstatehandler"); #include "clusterstatehandler.h" #include "iclusterstatechangedhandler.h" #include <vespa/vespalib/util/closuretask.h> -#include <algorithm> +#include <sstream> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.clusterstatehandler"); using storage::spi::Bucket; using storage::spi::BucketIdListResult; diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 665507b5bc0..95c005ec46a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "combiningfeedview.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/document/fieldvalue/document.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.combiningfeedview"); @@ -53,13 +53,13 @@ CombiningFeedView::CombiningFeedView(const std::vector<IFeedView::SP> &views, } -CombiningFeedView::~CombiningFeedView(void) +CombiningFeedView::~CombiningFeedView() { } const ISimpleDocumentMetaStore * -CombiningFeedView::getDocumentMetaStorePtr(void) const +CombiningFeedView::getDocumentMetaStorePtr() const { return NULL; } @@ -94,7 +94,7 @@ CombiningFeedView::findPrevDbdId(const document::GlobalId &gid, const DocumentTypeRepo::SP & -CombiningFeedView::getDocumentTypeRepo(void) const +CombiningFeedView::getDocumentTypeRepo() const { return _repo; } @@ -247,7 +247,7 @@ CombiningFeedView::heartBeat(search::SerialNum serialNum) void -CombiningFeedView::sync(void) +CombiningFeedView::sync() { getReadyFeedView()->sync(); // Assume this synced all feed views due to sharing of threads. diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index 879fc75a802..0b6b9538721 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -30,7 +30,7 @@ private: bool _clusterUp; bool _forceReady; - const ISimpleDocumentMetaStore * getDocumentMetaStorePtr(void) const override; + const ISimpleDocumentMetaStore * getDocumentMetaStorePtr() const override; void findPrevDbdId(const document::GlobalId &gid, DocumentOperation &op); uint32_t getReadyFeedViewId() const { return 0u; } @@ -63,7 +63,7 @@ public: virtual ~CombiningFeedView(); - const document::DocumentTypeRepo::SP & getDocumentTypeRepo(void) const override; + const document::DocumentTypeRepo::SP & getDocumentTypeRepo() const override; /** * Similar to IPersistenceHandler functions. diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp deleted file mode 100644 index 3238f67353c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "data_directory_upgrader.h" - -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/exceptions.h> -#include <iostream> -#include <sys/stat.h> - -namespace proton { - -namespace { - -vespalib::string UPGRADE_SOURCE_FILE = "data-directory-upgrade-source.txt"; -vespalib::string DOWNGRADE_SCRIPT_FILE = "data-directory-downgrade.sh"; - -bool -isValidDir(const vespalib::string &dir, char prefix) -{ - if (dir.empty() || dir[0] != prefix) { - return false; - } - vespalib::asciistream stream(dir.substr(1)); - uint32_t number = 0; - try { - stream >> number; - } catch (const vespalib::IllegalArgumentException &) { - return false; - } - return true; -} - -bool -isRowDir(const vespalib::string &dir) -{ - return isValidDir(dir, 'r'); -} - -bool -isColumnDir(const vespalib::string &dir) -{ - return isValidDir(dir, 'c'); -} - -vespalib::string -createDirString(const DataDirectoryUpgrader::RowColDirs &dirs) -{ - vespalib::asciistream result; - bool first = true; - for (const auto &dir : dirs) { - if (!first) { - result << ", "; - } - result << "'" << dir.dir() << "'"; - first = false; - } - return result.str(); -} - -void -writeUpgradeFile(const vespalib::string &srcDir, - const vespalib::string &dstDir) -{ - vespalib::File file(dstDir + "/" + UPGRADE_SOURCE_FILE); - file.open(vespalib::File::CREATE); - file.write(&srcDir[0], srcDir.size(), 0); - file.close(); -} - -void -writeDowngradeScript(const vespalib::string &scanDir, - const vespalib::string &dstDir, - const DataDirectoryUpgrader::RowColDir &rowColDir) -{ - vespalib::asciistream script; - vespalib::string fullRowDir = scanDir + "/" + rowColDir.row(); - vespalib::string fullRowColDir = scanDir + "/" + rowColDir.dir(); - script << "#!/bin/sh\n\n"; - script << "mkdir " << fullRowDir << " || exit 1\n"; - script << "chown yahoo " << fullRowDir << "\n"; - script << "mv " << dstDir << " " << fullRowColDir << "\n"; - script << "rm " << fullRowColDir << "/" << UPGRADE_SOURCE_FILE << "\n"; - script << "rm " << fullRowColDir << "/" << DOWNGRADE_SCRIPT_FILE << "\n"; - vespalib::string fileName = dstDir + "/" + DOWNGRADE_SCRIPT_FILE; - vespalib::File file(fileName); - file.open(vespalib::File::CREATE); - file.write(script.c_str(), script.size(), 0); - file.close(); - chmod(fileName.c_str(), 0755); -} - -} - - -DataDirectoryUpgrader::RowColDir::RowColDir(const vespalib::string &row_, - const vespalib::string &col_) - : _row(row_), - _col(col_) -{ -} - -DataDirectoryUpgrader::RowColDir::~RowColDir() { } - -DataDirectoryUpgrader::ScanResult::ScanResult() - : _rowColDirs(), - _destDirExisting(false) -{ -} - -DataDirectoryUpgrader::UpgradeResult::UpgradeResult(const Status status, - const vespalib::string &desc) - : _status(status), - _desc(desc) -{ -} - -DataDirectoryUpgrader::DataDirectoryUpgrader(const vespalib::string &scanDir, - const vespalib::string &destDir) - : _scanDir(scanDir), - _destDir(destDir) -{ -} - -DataDirectoryUpgrader::~DataDirectoryUpgrader() {} - -DataDirectoryUpgrader::ScanResult -DataDirectoryUpgrader::scan() const -{ - ScanResult result; - try { - vespalib::DirectoryList dirs = listDirectory(_scanDir); - for (const auto &dir : dirs) { - if (isRowDir(dir)) { - vespalib::DirectoryList subDirs = listDirectory(_scanDir + "/" + dir); - for (const auto &subDir : subDirs) { - if (isColumnDir(subDir)) { - result.addDir(RowColDir(dir, subDir)); - } - } - } - } - } catch (const vespalib::IoException &) { - // Scan dir does not exists - } - try { - if (vespalib::stat(_destDir).get() != NULL) { - result.setDestDirExisting(true); - } - } catch (const vespalib::IoException &) {} - std::sort(result.getRowColDirs().begin(), result.getRowColDirs().end()); - return result; -} - -DataDirectoryUpgrader::UpgradeResult -DataDirectoryUpgrader::upgrade(const ScanResult &scanResult) const -{ - if (scanResult.isDestDirExisting()) { - return UpgradeResult(IGNORE, - vespalib::make_string("Destination directory '%s' is already existing", - _destDir.c_str())); - } - const RowColDirs &rowColDirs = scanResult.getRowColDirs(); - if (rowColDirs.empty()) { - return UpgradeResult(IGNORE, "No directory to upgrade"); - } - if (rowColDirs.size() > 1) { - return UpgradeResult(ERROR, - vespalib::make_string("Can only upgrade a single directory, was asked to upgrade %zu (%s)", - rowColDirs.size(), createDirString(rowColDirs).c_str())); - } - const vespalib::string src = _scanDir + "/" + rowColDirs[0].dir(); - const vespalib::string &dst = _destDir; - try { - if (!vespalib::rename(src, dst)) { - return UpgradeResult(ERROR, - vespalib::make_string("Failed to rename directory '%s' to '%s'", - src.c_str(), dst.c_str())); - } - const vespalib::string rmDir = _scanDir + "/" + rowColDirs[0].row(); - if (!vespalib::rmdir(rmDir, false)) { - return UpgradeResult(ERROR, - vespalib::make_string("Failed to remove empty directory '%s'", - rmDir.c_str())); - } - writeUpgradeFile(src, dst); - writeDowngradeScript(_scanDir, dst, rowColDirs[0]); - } catch (const vespalib::IoException &ex) { - return UpgradeResult(ERROR, - vespalib::make_string("Got exception during data directory upgrade from '%s' to '%s': %s", - src.c_str(), dst.c_str(), ex.what())); - } - return UpgradeResult(COMPLETE, - vespalib::make_string("Moved data from '%s' to '%s'", - src.c_str(), dst.c_str())); -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h b/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h deleted file mode 100644 index 66963dd8f19..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/data_directory_upgrader.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace proton { - -/** - * Class used to upgrade a row column directory /rX/cY to an elastic directory /nZ - * where Z is the distribution key for that search node. - */ -class DataDirectoryUpgrader -{ -public: - class RowColDir - { - private: - vespalib::string _row; - vespalib::string _col; - - public: - RowColDir(const vespalib::string &row_, const vespalib::string &col_); - ~RowColDir(); - const vespalib::string &row() const { return _row; } - const vespalib::string &col() const { return _col; } - vespalib::string dir() const { return row() + "/" + col(); } - bool operator< (const RowColDir &rhs) const { return dir() < rhs.dir(); } - }; - typedef std::vector<RowColDir> RowColDirs; - - class ScanResult - { - private: - RowColDirs _rowColDirs; - bool _destDirExisting; - - public: - ScanResult(); - void addDir(const RowColDir &dir) { - _rowColDirs.push_back(dir); - } - RowColDirs &getRowColDirs() { return _rowColDirs; } - const RowColDirs &getRowColDirs() const { return _rowColDirs; } - void setDestDirExisting(bool val) { _destDirExisting = val; } - bool isDestDirExisting() const { return _destDirExisting; } - }; - - enum Status - { - IGNORE, - COMPLETE, - ERROR - }; - - class UpgradeResult - { - private: - const Status _status; - const vespalib::string _desc; - - public: - UpgradeResult(const Status status, const vespalib::string &desc); - Status getStatus() const { return _status; } - const vespalib::string &getDesc() const { return _desc; } - }; - -private: - const vespalib::string _scanDir; - const vespalib::string _destDir; - -public: - DataDirectoryUpgrader(const vespalib::string &scanDir, const vespalib::string &destDir); - ~DataDirectoryUpgrader(); - ScanResult scan() const; - UpgradeResult upgrade(const ScanResult &scanResult) const; -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp b/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp index 4b14fae358b..7ede0de3971 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.ddbstate"); - #include "ddbstate.h" - +#include <cassert> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h index 8c8bf0bd162..ace55498fce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h @@ -4,9 +4,9 @@ #include <vespa/vespalib/stllike/string.h> #include <mutex> #include <condition_variable> +#include <vector> -namespace proton -{ +namespace proton { /** * Track document db main state and validate that state transitions follow @@ -18,8 +18,7 @@ namespace proton class DDBState { public: - enum class State - { + enum class State { CONSTRUCT, LOAD, REPLAY_TRANSACTION_LOG, @@ -31,8 +30,7 @@ public: DEAD }; - enum class ConfigState - { + enum class ConfigState { OK, NEED_RESTART }; @@ -52,103 +50,53 @@ private: public: DDBState(); - ~DDBState(); /** * Try to enter LOAD state. Fail and return false if document db is * being shut down. */ - bool - enterLoadState(); + bool enterLoadState(); + bool enterReplayTransactionLogState(); + bool enterReplaySpoolerState(); + bool enterRedoReprocessState(); + bool enterApplyLiveConfigState(); + bool enterReprocessState(); + bool enterOnlineState(); + void enterShutdownState(); + void enterDeadState(); + State getState() const { return _state; } + static vespalib::string getStateString(State state); - bool - enterReplayTransactionLogState(); - - bool - enterReplaySpoolerState(); - - bool - enterRedoReprocessState(); - - bool - enterApplyLiveConfigState(); - - bool - enterReprocessState(); - - bool - enterOnlineState(); - - void - enterShutdownState(); - - void - enterDeadState(); - - State - getState(void) const - { - return _state; - } - - static vespalib::string - getStateString(State state); - - bool - getClosed(void) const - { + bool getClosed() const{ State state(_state); return state >= State::SHUTDOWN; } - bool - getAllowReconfig(void) const - { + bool getAllowReconfig() const { State state(_state); return state >= State::APPLY_LIVE_CONFIG && state < State::SHUTDOWN; } - bool - getAllowPrune(void) const - { + bool getAllowPrune() const { State state(_state); return state == State::ONLINE; } - static bool - getDelayedConfig(ConfigState state) - { + static bool getDelayedConfig(ConfigState state) { return state != ConfigState::OK; } - bool - getDelayedConfig() const - { + bool getDelayedConfig() const { ConfigState state(_configState); return getDelayedConfig(state); } - void - clearDelayedConfig(); - - ConfigState - getConfigState() const - { - return _configState; - } - - static vespalib::string - getConfigStateString(ConfigState configState); - - void - setConfigState(ConfigState newConfigState); - - void - waitForOnlineState(); + void clearDelayedConfig(); + ConfigState getConfigState() const { return _configState; } + static vespalib::string getConfigStateString(ConfigState configState); + void setConfigState(ConfigState newConfigState); + void waitForOnlineState(); }; - - } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp index a6c339076a5..47e6c84f172 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "disk_mem_usage_forwarder.h" #include <vespa/searchlib/common/lambdatask.h> +#include <cassert> using search::makeLambdaTask; -namespace proton -{ +namespace proton { DiskMemUsageForwarder::DiskMemUsageForwarder(searchcorespi::index::IThreadService &executor) : IDiskMemUsageNotifier(), diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.h index 28e4d3d0acf..26ce2c69d7c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.h @@ -2,12 +2,12 @@ #pragma once -#include <vespa/searchcorespi/index/i_thread_service.h> #include "i_disk_mem_usage_notifier.h" #include "i_disk_mem_usage_listener.h" +#include <vespa/searchcorespi/index/i_thread_service.h> +#include <vector> -namespace proton -{ +namespace proton { /** * Forwarder for disk/memory usage state changes. Notification is forwarded @@ -22,10 +22,10 @@ class DiskMemUsageForwarder : public IDiskMemUsageNotifier, void forward(DiskMemUsageState state); public: DiskMemUsageForwarder(searchcorespi::index::IThreadService &executor); - virtual ~DiskMemUsageForwarder(); - virtual void addDiskMemUsageListener(IDiskMemUsageListener *listener) override; - virtual void removeDiskMemUsageListener(IDiskMemUsageListener *listener) override; - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + ~DiskMemUsageForwarder() override; + void addDiskMemUsageListener(IDiskMemUsageListener *listener) override; + void removeDiskMemUsageListener(IDiskMemUsageListener *listener) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp index fc6b7b9eeee..200ced19d02 100644 --- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp @@ -59,7 +59,7 @@ DocStoreValidator::visit(uint32_t lid) void -DocStoreValidator::visitDone(void) +DocStoreValidator::visitDone() { _invalid->invalidateCachedCount(); _orphans->invalidateCachedCount(); @@ -68,13 +68,13 @@ DocStoreValidator::visitDone(void) } uint32_t -DocStoreValidator::getInvalidCount(void) const +DocStoreValidator::getInvalidCount() const { return _invalid->countTrueBits(); } uint32_t -DocStoreValidator::getOrphanCount(void) const +DocStoreValidator::getOrphanCount() const { return _orphans->countTrueBits(); } @@ -93,7 +93,7 @@ DocStoreValidator::killOrphans(search::IDocumentStore &store, LidVectorContext::SP -DocStoreValidator::getInvalidLids(void) const +DocStoreValidator::getInvalidLids() const { LidVectorContext::SP res(new LidVectorContext(_docIdLimit)); assert(_invalid->size() == _docIdLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h index f0eaa4bb01d..fd74ef45583 100644 --- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h +++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h @@ -24,7 +24,7 @@ public: virtual void visit(uint32_t lid, const document::Document &doc) override; virtual void visit(uint32_t lid) override; - void visitDone(void); + void visitDone(); void killOrphans(search::IDocumentStore &store, search::SerialNum serialNum); uint32_t getInvalidCount() const; uint32_t getOrphanCount() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index 535289886f6..d64b608a7d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "document_db_maintenance_config.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 2b724f41ae4..4b48de6fa5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -15,7 +15,7 @@ private: double _age; public: - DocumentDBPruneConfig(void); + DocumentDBPruneConfig(); DocumentDBPruneConfig(double interval, double age); bool operator==(const DocumentDBPruneConfig &rhs) const; @@ -32,11 +32,11 @@ private: double _interval; public: - DocumentDBHeartBeatConfig(void); + DocumentDBHeartBeatConfig(); DocumentDBHeartBeatConfig(double interval); bool operator==(const DocumentDBHeartBeatConfig &rhs) const; - double getInterval(void) const { return _interval; } + double getInterval() const { return _interval; } }; class DocumentDBLidSpaceCompactionConfig @@ -83,7 +83,7 @@ private: double _resourceLimitFactor; public: - DocumentDBMaintenanceConfig(void); + DocumentDBMaintenanceConfig(); DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneRemovedDocuments, const DocumentDBHeartBeatConfig &heartBeat, @@ -98,13 +98,13 @@ public: operator==(const DocumentDBMaintenanceConfig &rhs) const; const DocumentDBPruneRemovedDocumentsConfig & - getPruneRemovedDocumentsConfig(void) const + getPruneRemovedDocumentsConfig() const { return _pruneRemovedDocuments; } const DocumentDBHeartBeatConfig & - getHeartBeatConfig(void) const + getHeartBeatConfig() const { return _heartBeat; } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp index d5777d3c723..31dbe5bd6cd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.document_scan_iterator"); - #include "document_scan_iterator.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp index fffb40d1cdf..5a5f12c6210 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.document_subdb_collection_explorer"); #include "document_subdb_collection_explorer.h" - #include "document_subdb_explorer.h" using vespalib::slime::Inserter; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp index d370a735a37..86e0554014a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_initializer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_subdb_collection_initializer.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp index 68d51e13f2c..f67902bcb82 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.document_subdb_explorer"); #include "document_subdb_explorer.h" #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp index 77e71169f09..4cf5047d150 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_subdb_initializer_result.h" using searchcorespi::IIndexManager; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp index 815bfc75a70..e1ca1ece3d5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.documentbucketmover"); #include "documentbucketmover.h" #include "idocumentmovehandler.h" +#include "maintenancedocumentsubdb.h" #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> -#include "maintenancedocumentsubdb.h" using document::BucketId; using document::Document; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h index 5f8b5c8fe45..989e9fcdff4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h @@ -47,7 +47,7 @@ public: void moveDocuments(size_t maxDocsToMove); void cancel() { setBucketDone(); } bool bucketDone() const { return _bucketDone; } - const MaintenanceDocumentSubDB * getSource(void) const { return _source; } + const MaintenanceDocumentSubDB * getSource() const { return _source; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 307707b7c6a..092aab38ec8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -10,11 +10,9 @@ #include "lid_space_compaction_handler.h" #include "maintenance_jobs_injector.h" #include "reconfig_params.h" -#include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> -#include <vespa/searchcore/proton/common/schemautil.h> #include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/initializer/task_runner.h> @@ -28,6 +26,7 @@ #include <vespa/searchlib/attribute/configconverter.h> #include <vespa/searchlib/engine/docsumreply.h> #include <vespa/searchlib/engine/searchreply.h> +#include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> @@ -178,15 +177,12 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _feedHandler.setBucketDBHandler(&_subDBs.getBucketDBHandler()); saveInitialConfig(*configSnapshot); resumeSaveConfig(); - SerialNum configSerial = _config_store->getPrevValidSerial( - _feedHandler.getPrunedSerialNum() + 1); + SerialNum configSerial = _config_store->getPrevValidSerial(_feedHandler.getPrunedSerialNum() + 1); assert(configSerial > 0); DocumentDBConfig::SP loaded_config; - _config_store->loadConfig(*configSnapshot, configSerial, - loaded_config); + _config_store->loadConfig(*configSnapshot, configSerial, loaded_config); // Grab relevant parts from pending config - loaded_config = DocumentDBConfigScout::scout(loaded_config, - *_pendingConfigSnapshot.get()); + loaded_config = DocumentDBConfigScout::scout(loaded_config, *_pendingConfigSnapshot.get()); // Ignore configs that are not relevant during replay of transaction log loaded_config = DocumentDBConfig::makeReplayConfig(loaded_config); @@ -198,13 +194,10 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _clusterStateHandler.addClusterStateChangedHandler(&_bucketHandler); for (auto subDb : _subDBs) { _lidSpaceCompactionHandlers.push_back(ILidSpaceCompactionHandler::UP - (new LidSpaceCompactionHandler(*subDb, - _docTypeName.getName()))); + (new LidSpaceCompactionHandler(*subDb, _docTypeName.getName()))); } - _writeFilter.setConfig(loaded_config->getMaintenanceConfigSP()-> - getAttributeUsageFilterConfig()); - fastos::TimeStamp visibilityDelay = - loaded_config->getMaintenanceConfigSP()->getVisibilityDelay(); + _writeFilter.setConfig(loaded_config->getMaintenanceConfigSP()->getAttributeUsageFilterConfig()); + fastos::TimeStamp visibilityDelay = loaded_config->getMaintenanceConfigSP()->getVisibilityDelay(); _visibility.setVisibilityDelay(visibilityDelay); if (_visibility.getVisibilityDelay() > 0) { _writeService.setTaskLimit(semiUnboundTaskLimit(_semiUnboundExecutorTaskLimit, _indexingThreads)); @@ -285,12 +278,10 @@ DocumentDB::initManagers() _initConfigSnapshot.reset(); InitializerTask::SP rootTask = _subDBs.createInitializer(*configSnapshot, _initConfigSerialNum, - _protonSummaryCfg, - _protonIndexCfg); + _protonSummaryCfg, _protonIndexCfg); InitializeThreads initializeThreads = _initializeThreads; _initializeThreads.reset(); - std::shared_ptr<TaskRunner> taskRunner(std::make_shared<TaskRunner> - (*initializeThreads)); + std::shared_ptr<TaskRunner> taskRunner(std::make_shared<TaskRunner>(*initializeThreads)); auto doneTask = std::make_unique<InitDoneTask>(std::move(initializeThreads), taskRunner, std::move(configSnapshot), *this); taskRunner->runTask(rootTask, _writeService.master(), std::move(doneTask)); @@ -300,8 +291,7 @@ void DocumentDB::initFinish(DocumentDBConfig::SP configSnapshot) { // Called by executor thread - _bucketHandler.setReadyBucketHandler( - _subDBs.getReadySubDB()->getDocumentMetaStoreContext().get()); + _bucketHandler.setReadyBucketHandler(_subDBs.getReadySubDB()->getDocumentMetaStoreContext().get()); _subDBs.initViews(*configSnapshot, _sessionManager); _syncFeedViewEnabled = true; syncFeedView(); @@ -651,9 +641,8 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) _config_store->saveConfig(configSnapshot, confSerial); } - void -DocumentDB::resumeSaveConfig(void) +DocumentDB::resumeSaveConfig() { SerialNum bestSerial = _config_store->getBestSerialNum(); if (bestSerial == 0) @@ -692,14 +681,12 @@ DocumentDB::onPerformPrune(SerialNum flushedSerial) _config_store->prune(flushedSerial); } - bool -DocumentDB::getAllowPrune(void) const +DocumentDB::getAllowPrune() const { return _state.getAllowPrune(); } - void DocumentDB::start() { @@ -710,14 +697,12 @@ DocumentDB::start() internalInit(); } - void DocumentDB::waitForInitDone() { _initGate.await(); } - void DocumentDB::startTransactionLogReplay() { @@ -737,9 +722,7 @@ DocumentDB::startTransactionLogReplay() *_config_store); _initGate.countDown(); - LOG(debug, - "DocumentDB(%s): Database started.", - _docTypeName.toString().c_str()); + LOG(debug, "DocumentDB(%s): Database started.", _docTypeName.toString().c_str()); } BucketGuard::UP DocumentDB::lockBucket(const document::BucketId &bucket) @@ -814,15 +797,13 @@ DocumentDB::setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum ser // TODO: Adjust tune. } - void DocumentDB::reconfigure(const DocumentDBConfig::SP & snapshot) { masterExecute([this, snapshot]() { newConfigSnapshot(snapshot); }); // Wait for config to be applied, or for document db close std::unique_lock<std::mutex> guard(_configMutex); - while ((_activeConfigSnapshotGeneration < snapshot->getGeneration()) && - !_state.getClosed()) { + while ((_activeConfigSnapshotGeneration < snapshot->getGeneration()) && !_state.getClosed()) { _configCV.wait(guard); } } @@ -880,21 +861,17 @@ DocumentDB::reportStatus() const float progress = _feedHandler.getReplayProgress() * 100.0f; vespalib::string msg = vespalib::make_string("DocumentDB replay transaction log on startup (%u%% done)", static_cast<uint32_t>(progress)); - return StatusReport::create(params.state(StatusReport::PARTIAL). - progress(progress). - message(msg)); + return StatusReport::create(params.state(StatusReport::PARTIAL).progress(progress).message(msg)); } else if (rawState == DDBState::State::APPLY_LIVE_CONFIG) { - return StatusReport::create(params.state(StatusReport::PARTIAL). - message("DocumentDB apply live config on startup")); + return StatusReport::create(params.state(StatusReport::PARTIAL) + .message("DocumentDB apply live config on startup")); } else if (rawState == DDBState::State::REPROCESS || rawState == DDBState::State::REDO_REPROCESS) { float progress = _subDBs.getReprocessingProgress() * 100.0f; vespalib::string msg = make_string("DocumentDB reprocess on startup (%u%% done)", - static_cast<uint32_t>(progress)); - return StatusReport::create(params.state(StatusReport::PARTIAL). - progress(progress). - message(msg)); + static_cast<uint32_t>(progress)); + return StatusReport::create(params.state(StatusReport::PARTIAL).progress(progress).message(msg)); } else if (_state.getDelayedConfig()) { return StatusReport::create(params.state(StatusReport::PARTIAL). message("DocumentDB delaying attribute aspects changes in config")); @@ -928,34 +905,13 @@ DocumentDB::replayConfig(search::SerialNum serialNum) _docTypeName.toString().c_str(), serialNum); } -void -DocumentDB::listSchema(std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations) -{ - DocumentDBConfig::SP activeSnapshot = getActiveConfig(); - if (activeSnapshot.get() == NULL || - activeSnapshot->getSchemaSP().get() == NULL) - { - return; - } - SchemaUtil::listSchema(*activeSnapshot->getSchemaSP(), - fieldNames, - fieldDataTypes, - fieldCollectionTypes, - fieldLocations); -} - - int64_t DocumentDB::getActiveGeneration() const { lock_guard guard(_configMutex); return _activeConfigSnapshotGeneration; } - void -DocumentDB::syncFeedView(void) +DocumentDB::syncFeedView() { // Called by executor or while in rendezvous with executor @@ -973,14 +929,12 @@ DocumentDB::syncFeedView(void) performDropFeedView(oldFeedView); } - bool DocumentDB::hasDocument(const document::DocumentId &id) { return _subDBs.getReadySubDB()->hasDocument(id); } - void DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) { @@ -1009,7 +963,7 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) } void -DocumentDB::performStartMaintenance(void) +DocumentDB::performStartMaintenance() { // Called by executor thread // Only start once, after replay done @@ -1030,13 +984,13 @@ DocumentDB::performStartMaintenance(void) } void -DocumentDB::stopMaintenance(void) +DocumentDB::stopMaintenance() { _maintenanceController.stop(); } void -DocumentDB::forwardMaintenanceConfig(void) +DocumentDB::forwardMaintenanceConfig() { // Called by executor thread DocumentDBConfig::SP activeConfig = getActiveConfig(); @@ -1053,8 +1007,7 @@ DocumentDB::forwardMaintenanceConfig(void) } void -DocumentDB::notifyClusterStateChanged( - const IBucketStateCalculator::SP &newCalc) +DocumentDB::notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) { // Called by executor thread _calc = newCalc; // Save for maintenance job injection @@ -1062,8 +1015,7 @@ DocumentDB::notifyClusterStateChanged( IFeedView::SP feedView(_feedView.get()); if (feedView.get() != NULL) { // Try downcast to avoid polluting API - CombiningFeedView *cfv = dynamic_cast<CombiningFeedView *> - (feedView.get()); + CombiningFeedView *cfv = dynamic_cast<CombiningFeedView *>(feedView.get()); if (cfv != NULL) cfv->setCalculator(newCalc); } @@ -1099,13 +1051,6 @@ DocumentDB::notifyAllBucketsChanged() _clusterStateHandler, "notready"); } - -searchcorespi::IIndexManagerFactory::SP -DocumentDB::getIndexManagerFactory(const vespalib::stringref &name) const -{ - return _owner.getIndexManagerFactory(name); -} - namespace { void @@ -1129,8 +1074,7 @@ struct TempAttributeMetric TempAttributeMetric() : _memoryUsage(), _bitVectors(0) - { - } + {} }; struct TempAttributeMetrics @@ -1376,12 +1320,9 @@ DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) _jobTrackers.updateMetrics(metrics.job); updateMetrics(metrics.attribute); - updateDocumentStoreMetrics(metrics.ready.documentStore, - _subDBs.getReadySubDB()); - updateDocumentStoreMetrics(metrics.removed.documentStore, - _subDBs.getRemSubDB()); - updateDocumentStoreMetrics(metrics.notReady.documentStore, - _subDBs.getNotReadySubDB()); + updateDocumentStoreMetrics(metrics.ready.documentStore, _subDBs.getReadySubDB()); + updateDocumentStoreMetrics(metrics.removed.documentStore, _subDBs.getRemSubDB()); + updateDocumentStoreMetrics(metrics.notReady.documentStore, _subDBs.getNotReadySubDB()); DocumentMetaStoreReadGuards dmss(_subDBs); updateLidSpaceMetrics(metrics.ready.lidSpace, dmss.readydms->get()); updateLidSpaceMetrics(metrics.notReady.lidSpace, dmss.notreadydms->get()); @@ -1391,13 +1332,10 @@ DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) void DocumentDB::sync(SerialNum syncTo) { - LOG(spam, - "DocumentDB(%s): sync(): serialNum=%" PRIu64, - _docTypeName.toString().c_str(), syncTo); + LOG(spam, "DocumentDB(%s): sync(): serialNum=%zu", _docTypeName.toString().c_str(), syncTo); _feedHandler.syncTls(syncTo); } - void DocumentDB::waitForOnlineState() { @@ -1410,5 +1348,4 @@ DocumentDB::getDistributionKey() const return _owner.getDistributionKey(); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index da933faeeb0..1fe634fcf4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once - #include "buckethandler.h" #include "clusterstatehandler.h" #include "configstore.h" @@ -18,6 +17,7 @@ #include "executorthreadingservice.h" #include "visibilityhandler.h" #include "i_document_subdb_owner.h" +#include "disk_mem_usage_forwarder.h" #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> @@ -30,24 +30,15 @@ #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> -#include "disk_mem_usage_forwarder.h" #include <vespa/metrics/updatehook.h> #include <mutex> #include <condition_variable> -namespace search -{ - -namespace common -{ - -class FileHeaderContext; - +namespace search { + namespace common { class FileHeaderContext; } + namespace transactionlog { class TransLogClient; } } -namespace transactionlog { class TransLogClient; } -} // namespace search - namespace proton { class IDocumentDBOwner; class MetricsWireService; @@ -163,7 +154,7 @@ private: /** * Resume interrupted config save if needed. */ - void resumeSaveConfig(void); + void resumeSaveConfig(); void setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum); @@ -195,7 +186,7 @@ private: /** * Implements IFeedHandlerOwner **/ - virtual bool getAllowPrune(void) const override; + virtual bool getAllowPrune() const override; void startTransactionLogReplay(); @@ -301,7 +292,7 @@ public: * * @return metrics update hook **/ - metrics::UpdateHook & getMetricsUpdateHook(void) { + metrics::UpdateHook & getMetricsUpdateHook() { return _metricsHook; } @@ -337,12 +328,12 @@ public: /** * Returns the feed handler for this database. */ - FeedHandler & getFeedHandler(void) { return _feedHandler; } + FeedHandler & getFeedHandler() { return _feedHandler; } /** * Returns the bucket handler for this database. */ - BucketHandler & getBucketHandler(void) { return _bucketHandler; } + BucketHandler & getBucketHandler() { return _bucketHandler; } /** * Returns the cluster state handler for this database. @@ -404,13 +395,7 @@ public: */ virtual void replayConfig(SerialNum serialNum) override; - const DocTypeName & getDocTypeName(void) const { return _docTypeName; } - - void - listSchema(std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations); + const DocTypeName & getDocTypeName() const { return _docTypeName; } void newConfigSnapshot(DocumentDBConfig::SP snapshot); @@ -422,9 +407,6 @@ public: // Implements IDocumentSubDBOwner void syncFeedView() override; - std::shared_ptr<searchcorespi::IIndexManagerFactory> - getIndexManagerFactory(const vespalib::stringref & name) const override; - vespalib::string getName() const override { return _docTypeName.getName(); } uint32_t getDistributionKey() const override; @@ -432,9 +414,9 @@ public: * Implements IFeedHandlerOwner **/ void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); - void performStartMaintenance(void); - void stopMaintenance(void); - void forwardMaintenanceConfig(void); + void performStartMaintenance(); + void stopMaintenance(); + void forwardMaintenanceConfig(); /** * Updates metrics collection object, and resets executor stats. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index c766318a294..4a645ea7888 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -9,6 +9,7 @@ #include <vespa/config-summarymap.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/config-documenttypes.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h> #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 46a6a8e76db..757259411e6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -6,7 +6,6 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchcommon/common/schema.h> -#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/proton/matching/ranking_constants.h> #include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> @@ -144,7 +143,7 @@ public: const vespalib::string &getDocTypeName() const { return _docTypeName; } - int64_t getGeneration(void) const { return _generation; } + int64_t getGeneration() const { return _generation; } const RankProfilesConfig &getRankProfilesConfig() const { return *_rankProfiles; } const RankingConstants &getRankingConstants() const { return *_rankingConstants; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 52d75e1ff63..8c737f26af7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/time/time_box.h> LOG_SETUP(".proton.server.documentdbconfigmanager"); @@ -351,7 +352,7 @@ DocumentDBConfigHelper::nextGeneration(int timeoutInMillis) } DocumentDBConfig::SP -DocumentDBConfigHelper::getConfig(void) const +DocumentDBConfigHelper::getConfig() const { return _mgr.getConfig(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h index cb74350684f..be132509e45 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.h @@ -42,7 +42,7 @@ public: DocumentDBConfig::SP getConfig() const; void forwardConfig(const BootstrapConfigSP & config); - const config::ConfigKeySet createConfigKeySet(void) const; + const config::ConfigKeySet createConfigKeySet() const; void setExtraConfigKeys(const config::ConfigKeySet & extraConfigKeys) { _extraConfigKeys = extraConfigKeys; } const config::ConfigKeySet & getExtraConfigKeys() const { return _extraConfigKeys; } const vespalib::string & getConfigId() const { return _configId; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp index 2ebadce7e38..53620c23dbf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "documentdbconfigscout.h" #include <vespa/searchcore/proton/attribute/attributesconfigscout.h> using vespa::config::search::AttributesConfig; -namespace proton -{ +namespace proton { DocumentDBConfig::SP diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index 09c68c567fa..e40d6480cc3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -2,6 +2,7 @@ #include "documentretriever.h" #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchcore/proton/attribute/document_field_retriever.h> #include <vespa/vespalib/geo/zcurve.h> @@ -138,7 +139,7 @@ void DocumentRetriever::populate(DocumentIdT lid, Document & doc) const } const IAttributeManager * -DocumentRetriever::getAttrMgr(void) const +DocumentRetriever::getAttrMgr() const { return &_attr_manager; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.h b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h index 7873c17401f..db085217a5d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h @@ -34,7 +34,7 @@ private: PositionFields _possiblePositionFields; AttributeFields _attributeFields; - const search::IAttributeManager * getAttrMgr(void) const override; + const search::IAttributeManager * getAttrMgr() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp index 6f7e4423396..63cfded1962 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp @@ -2,6 +2,7 @@ #include "documentretrieverbase.h" #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/stllike/lrucache_map.hpp> using document::DocumentId; @@ -61,7 +62,7 @@ DocumentRetrieverBase::getDocumentMetaData(const DocumentId &id) const { const search::IAttributeManager * -DocumentRetrieverBase::getAttrMgr(void) const +DocumentRetrieverBase::getAttrMgr() const { return NULL; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h index 63fcdcd1f98..d28b3e0ead2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h @@ -25,7 +25,7 @@ class DocumentRetrieverBase : public IDocumentRetriever const bool _hasFields; protected: - virtual const search::IAttributeManager * getAttrMgr(void) const; + virtual const search::IAttributeManager * getAttrMgr() const; public: DocumentRetrieverBase(const DocTypeName &docTypeName, const document::DocumentTypeRepo &repo, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index d1100613c23..cf085feaad9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -1,28 +1,27 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/varholder.h> -#include <vespa/searchcore/config/config-proton.h> -#include <vespa/searchlib/common/serialnum.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> +#include <vespa/searchcore/config/config-proton.h> +#include <vespa/searchlib/common/serialnum.h> +#include <vespa/vespalib/util/varholder.h> #include <mutex> -namespace vespalib -{ -class Clock; -class ThreadExecutor; -class ThreadStackExecutorBase; +namespace vespalib { + class Clock; + class ThreadExecutor; + class ThreadStackExecutorBase; } namespace search { -namespace common { class FileHeaderContext; } -namespace transactionlog { class SyncProxy; } + namespace common { class FileHeaderContext; } + namespace transactionlog { class SyncProxy; } } namespace searchcorespi { -class IFlushTarget; -namespace index { class IThreadingService; } + class IFlushTarget; + namespace index { class IThreadingService; } } namespace proton { @@ -44,8 +43,8 @@ class IRreprocessingTask; class ReconfigParams; namespace matching { -class QueryLimiter; -class SessionManager; + class QueryLimiter; + class SessionManager; } namespace initializer { class InitializerTask; } @@ -131,11 +130,11 @@ public: initViews(const DocumentDBConfig &configSnapshot, const SessionManagerSP &sessionManager); - void clearViews(void); - void onReplayDone(void); + void clearViews(); + void onReplayDone(); void onReprocessDone(SerialNum serialNum); - SerialNum getOldestFlushedSerial(void); - SerialNum getNewestFlushedSerial(void); + SerialNum getOldestFlushedSerial(); + SerialNum getNewestFlushedSerial(); void pruneRemovedFields(SerialNum serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp index 6c975c5ae35..bde5978ddd8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "emptysearchview.h" #include <vespa/searchlib/engine/docsumreply.h> #include <vespa/searchlib/engine/searchreply.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.emptysearchview"); @@ -12,8 +12,7 @@ using search::engine::DocsumRequest; using search::engine::SearchReply; using search::engine::SearchRequest; -namespace proton -{ +namespace proton { EmptySearchView::EmptySearchView() : ISearchHandler() diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h index 68d7a2e9190..7b7378dea45 100644 --- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h @@ -11,7 +11,7 @@ class EmptySearchView : public ISearchHandler public: typedef std::shared_ptr<EmptySearchView> SP; - EmptySearchView(void); + EmptySearchView(); /** * Implements ISearchHandler diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 2c7627ea712..fb65b09b309 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.executor_thread_service"); #include "executor_thread_service.h" #include <vespa/vespalib/util/closuretask.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 99df6ed8e1b..b04a1047934 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.executorthreadingservice"); #include "executorthreadingservice.h" #include <vespa/vespalib/util/executor.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index e1a9a2f421c..67d869f41e3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -82,6 +82,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co _attributeGrow, _attributeGrowNumDocs, _fastAccessAttributesOnly, + _writeService.master(), attrMgrResult); } diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index e755731c0a9..04a7a6d6407 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.fast_access_doc_subdb_configurer"); - #include "fast_access_doc_subdb_configurer.h" #include "i_attribute_writer_factory.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 9fa9ae27206..aa1541d766e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -12,15 +12,13 @@ #include <vespa/documentapi/messagebus/messages/feedreply.h> #include <vespa/documentapi/messagebus/messages/removedocumentreply.h> #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> -#include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/persistenceengine/transport_latch.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.feedhandler"); @@ -591,7 +589,7 @@ FeedHandler::flushDone(SerialNum flushedSerial) flushedSerial))); } -void FeedHandler::changeToNormalFeedState(void) { +void FeedHandler::changeToNormalFeedState() { changeFeedState(FeedState::SP(new NormalState(*this))); } @@ -743,7 +741,7 @@ FeedHandler::handleMove(MoveOperation &op) void -FeedHandler::heartBeat(void) +FeedHandler::heartBeat() { assert(_writeService.master().isCurrentThread()); _activeFeedView->heartBeat(_serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 726039e0be7..0d81cb2f6ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -285,10 +285,10 @@ public: * Implements IHeartBeatHandler */ virtual void - heartBeat(void) override; + heartBeat() override; virtual void - sync(void); + sync(); /** * Implements TransLogClient::Session::Callback. @@ -297,10 +297,10 @@ public: receive(const Packet &packet) override; virtual void - eof(void) override; + eof() override; virtual void - inSync(void) override; + inSync() override; /** * Implements IPruneRemovedDocumentsHandler diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 80c8b8adac1..b444cd99952 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -6,14 +6,14 @@ #include <vespa/config/print/fileconfigsnapshotreader.h> #include <vespa/config/print/fileconfigsnapshotwriter.h> #include <vespa/searchcommon/common/schemaconfigurer.h> -#include <vespa/searchlib/index/schemautil.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/config-summarymap.h> #include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <fstream> #include <vespa/fastos/file.h> #include <vespa/config/helper/configgetter.hpp> +#include <fstream> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.server.fileconfigmanager"); @@ -268,13 +268,13 @@ FileConfigManager::FileConfigManager(const vespalib::string &baseDir, } -FileConfigManager::~FileConfigManager(void) +FileConfigManager::~FileConfigManager() { } SerialNum -FileConfigManager::getBestSerialNum(void) const +FileConfigManager::getBestSerialNum() const { Snapshot snap = _info.getBestSnapshot(); return snap.valid ? snap.syncToken : UINT64_C(0); @@ -282,7 +282,7 @@ FileConfigManager::getBestSerialNum(void) const SerialNum -FileConfigManager::getOldestSerialNum(void) const +FileConfigManager::getOldestSerialNum() const { SerialNum res = 0; const SnapshotList &snaps = _info.snapshots(); @@ -410,7 +410,7 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, void -FileConfigManager::removeInvalid(void) +FileConfigManager::removeInvalid() { typedef std::vector<SerialNum> RemVec; RemVec toRem; diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h index eb8b1a8c690..a8dfc139d4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h @@ -34,7 +34,7 @@ public: const vespalib::string &docTypeName); virtual - ~FileConfigManager(void); + ~FileConfigManager(); virtual SerialNum getBestSerialNum() const override; virtual SerialNum getOldestSerialNum() const override; diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp index 6ecdbf8e51d..3dfe5d95e59 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "flushhandlerproxy.h" #include "documentdb.h" -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.flushhandlerproxy"); using searchcorespi::IFlushTarget; @@ -18,21 +15,21 @@ FlushHandlerProxy::FlushHandlerProxy(const DocumentDB::SP &documentDB) } -FlushHandlerProxy::~FlushHandlerProxy(void) +FlushHandlerProxy::~FlushHandlerProxy() { _documentDB->release(); } std::vector<IFlushTarget::SP> -FlushHandlerProxy::getFlushTargets(void) +FlushHandlerProxy::getFlushTargets() { return _documentDB->getFlushTargets(); } IFlushHandler::SerialNum -FlushHandlerProxy::getCurrentSerialNumber(void) const +FlushHandlerProxy::getCurrentSerialNumber() const { return _documentDB->getCurrentSerialNumber(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp index d382a76faeb..33d4ccb4ef0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "forcecommitcontext.h" #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/common/docid_limit.h> +#include <cassert> -namespace proton -{ - +namespace proton { ForceCommitContext::ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore) @@ -18,7 +16,6 @@ ForceCommitContext::ForceCommitContext(vespalib::Executor &executor, { } - ForceCommitContext::~ForceCommitContext() { if (_docIdLimit != nullptr) { @@ -30,21 +27,18 @@ ForceCommitContext::~ForceCommitContext() } } - void ForceCommitContext::reuseLids(std::vector<uint32_t> &&lids) { _task->reuseLids(std::move(lids)); } - void ForceCommitContext::holdUnblockShrinkLidSpace() { _task->holdUnblockShrinkLidSpace(); } - void ForceCommitContext::registerCommittedDocIdLimit(uint32_t committedDocIdLimit, DocIdLimit *docIdLimit) @@ -53,5 +47,4 @@ ForceCommitContext::registerCommittedDocIdLimit(uint32_t committedDocIdLimit, _docIdLimit = docIdLimit; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h index ecf50f1402a..65a677627ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h @@ -3,17 +3,13 @@ #pragma once #include <vespa/searchlib/common/idestructorcallback.h> +#include <memory> +#include <cstdint> +#include <vector> -namespace vespalib -{ - -class Executor; - -} - +namespace vespalib { class Executor; } -namespace proton -{ +namespace proton { class ForceCommitDoneTask; class IDocumentMetaStore; @@ -30,22 +26,18 @@ class ForceCommitContext : public search::IDestructorCallback { vespalib::Executor &_executor; std::unique_ptr<ForceCommitDoneTask> _task; - uint32_t _committedDocIdLimit; + uint32_t _committedDocIdLimit; DocIdLimit *_docIdLimit; public: ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore); - virtual ~ForceCommitContext(); + ~ForceCommitContext() override; void reuseLids(std::vector<uint32_t> &&lids); - void holdUnblockShrinkLidSpace(); - - void registerCommittedDocIdLimit(uint32_t committedDocIdLimit, - DocIdLimit *docIdLimit); + void registerCommittedDocIdLimit(uint32_t committedDocIdLimit, DocIdLimit *docIdLimit); }; - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp index f78eed0164e..d669cd5a2a7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> -namespace proton -{ - +namespace proton { ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore) : _lidsToReuse(), @@ -15,12 +12,10 @@ ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore) { } - ForceCommitDoneTask::~ForceCommitDoneTask() { } - void ForceCommitDoneTask::reuseLids(std::vector<uint32_t> &&lids) { @@ -28,7 +23,6 @@ ForceCommitDoneTask::reuseLids(std::vector<uint32_t> &&lids) _lidsToReuse = std::move(lids); } - void ForceCommitDoneTask::run() { @@ -44,5 +38,4 @@ ForceCommitDoneTask::run() } } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h index 73cb2ee1b47..28190ff68d5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h @@ -3,9 +3,9 @@ #pragma once #include <vespa/vespalib/util/executor.h> +#include <vector> -namespace proton -{ +namespace proton { class IDocumentMetaStore; @@ -32,7 +32,7 @@ class ForceCommitDoneTask : public vespalib::Executor::Task public: ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore); - virtual ~ForceCommitDoneTask(); + ~ForceCommitDoneTask() override; void reuseLids(std::vector<uint32_t> &&lids); @@ -40,12 +40,11 @@ public: _holdUnblockShrinkLidSpace = true; } - virtual void run() override; + void run() override; bool empty() const { return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace; } }; - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp b/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp index be5ed968851..8ab35d08d7e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/health_adapter.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.health_adapter"); -#include <vespa/vespalib/util/stringfmt.h> #include "health_adapter.h" +#include <vespa/vespalib/util/stringfmt.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp index b9dcf94c105..92c417d405b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/heart_beat_job.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "heart_beat_job.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h index 20b2529d991..1dfa2a0dee3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h @@ -4,8 +4,6 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> -namespace searchcorespi { class IIndexManagerFactory; } - namespace proton { /** @@ -17,8 +15,6 @@ class IDocumentSubDBOwner public: virtual ~IDocumentSubDBOwner() {} virtual void syncFeedView() = 0; - virtual std::shared_ptr<searchcorespi::IIndexManagerFactory> - getIndexManagerFactory(const vespalib::stringref &name) const = 0; virtual vespalib::string getName() const = 0; virtual uint32_t getDistributionKey() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp index 0fcf29b50c6..0957b74f778 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "idocumentdbowner.h" -namespace proton -{ +namespace proton { -IDocumentDBOwner::~IDocumentDBOwner(void) +IDocumentDBOwner::~IDocumentDBOwner() { } diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h index 6a9e8702495..0052b95234f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentdbowner.h @@ -2,26 +2,21 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> +#include <memory> +#include <cstdint> -namespace proton -{ +namespace proton { class IDocumentDBReferenceRegistry; class IDocumentDBOwner { public: - virtual ~IDocumentDBOwner(void); + virtual ~IDocumentDBOwner(); virtual bool isInitializing() const = 0; - - virtual searchcorespi::IIndexManagerFactory::SP - getIndexManagerFactory(const vespalib::stringref & name) const = 0; virtual uint32_t getDistributionKey() const = 0; virtual std::shared_ptr<IDocumentDBReferenceRegistry> getDocumentDBReferenceRegistry() const = 0; }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 86d8cdc01e0..5fd480f6c03 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -8,21 +8,19 @@ #include <vespa/searchlib/util/searchable_stats.h> -namespace search { - namespace index { - class Schema; - } +namespace search::index { + class Schema; } namespace document { class DocumentId; } namespace searchcorespi { -class IFlushTarget; -class IIndexManagerFactory; -class IIndexManager; + class IFlushTarget; + class IIndexManager; } namespace proton { - namespace matching { class SessionManager; } + +namespace matching { class SessionManager; } class DocumentDBConfig; class DocumentSubDbInitializer; diff --git a/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h b/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h index 705f7ec8408..7ad0e218c85 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ifrozenbuckethandler.h @@ -22,7 +22,7 @@ public: document::BucketId _bucketId; }; - virtual ~IFrozenBucketHandler(void) { } + virtual ~IFrozenBucketHandler() { } virtual ExclusiveBucketGuard::UP acquireExclusiveBucket(document::BucketId bucket) = 0; virtual void addListener(IBucketFreezeListener *listener) = 0; virtual void removeListener(IBucketFreezeListener *listener) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h b/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h index f863bcc7b4a..a9391f680d5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/iheartbeathandler.h @@ -11,10 +11,10 @@ class IHeartBeatHandler { public: virtual void - heartBeat(void) = 0; + heartBeat() = 0; virtual - ~IHeartBeatHandler(void) + ~IHeartBeatHandler() { } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h b/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h index 3ff8dfcab2e..187ca882778 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ipruneremoveddocumentshandler.h @@ -14,7 +14,7 @@ public: performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) = 0; virtual - ~IPruneRemovedDocumentsHandler(void) + ~IPruneRemovedDocumentsHandler() { } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp index 16ed662b310..c798d56fc3d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "ireplayconfig.h" -namespace proton -{ +namespace proton { -IReplayConfig::~IReplayConfig(void) +IReplayConfig::~IReplayConfig() { } diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h index 6da62888208..9d15ad8acc4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h @@ -11,7 +11,7 @@ class IReplayConfig { public: virtual - ~IReplayConfig(void); + ~IReplayConfig(); virtual void replayConfig(search::SerialNum serialNum) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h index 165af7c3eee..edae33e45c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h +++ b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h @@ -30,7 +30,7 @@ public: heartBeat(search::SerialNum serialNum) = 0; virtual const search::IDocumentStore & - getDocumentStore(void) const = 0; + getDocumentStore() const = 0; virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid, diff --git a/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h b/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h index f8c79d6485d..3f65e1b2d2f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h +++ b/searchcore/src/vespa/searchcore/proton/server/itlssyncer.h @@ -12,7 +12,7 @@ namespace proton class ITlsSyncer { public: - virtual ~ITlsSyncer(void) = default; + virtual ~ITlsSyncer() = default; virtual void sync() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp index d8c197b6c87..cc61f34ee79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/job_tracked_maintenance_job.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.job_tracked_maintenance_job"); #include "job_tracked_maintenance_job.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 0ce8cfe9575..3d655f3decf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -1,14 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.lid_space_compaction_job"); - #include "lid_space_compaction_job.h" -#include "ifrozenbuckethandler.h" -#include <vespa/searchcore/proton/common/eventlogger.h> #include "imaintenancejobrunner.h" #include "i_disk_mem_usage_notifier.h" +#include <vespa/searchcore/proton/common/eventlogger.h> +#include <cassert> using search::DocumentMetaData; using search::LidUsageStats; @@ -77,9 +73,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); _opStorer.storeOperation(op); _handler.handleCompactLidSpace(op); - if (LOG_WOULD_LOG(event)) { - EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); - } + EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); _shouldCompactLidSpace = false; } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp index 3bc7ae7603e..cd3a58c791d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.maintenance_controller_explorer"); #include "maintenance_controller_explorer.h" - #include <vespa/vespalib/data/slime/cursor.h> using vespalib::slime::Cursor; diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index 7bec456d0f5..8e9b711f627 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -132,7 +132,7 @@ MaintenanceController::performClearJobs() void -MaintenanceController::stop(void) +MaintenanceController::stop() { assert(!_masterThread.isCurrentThread()); _stopping = true; @@ -160,7 +160,7 @@ MaintenanceController::start(const DocumentDBMaintenanceConfig::SP &config) void -MaintenanceController::restart(void) +MaintenanceController::restart() { // Called by master write thread if (!_started || _stopping || !_readySubDB.valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 86ee2fe2ac0..c567930d68d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -4,6 +4,7 @@ #include "searchcontext.h" #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/searchreply.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.matchview"); diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h index 984695867e6..9878205f354 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h @@ -43,7 +43,7 @@ public: const IAttributeManager::SP & getAttributeManager() const { return _attrMgr; } const matching::SessionManager::SP & getSessionManager() const { return _sessionMgr; } const IDocumentMetaStoreContext::SP & getDocumentMetaStore() const { return _metaStore; } - DocIdLimit & getDocIdLimit(void) const { return _docIdLimit; } + DocIdLimit & getDocIdLimit() const { return _docIdLimit; } // Throws on error. matching::Matcher::SP getMatcher(const vespalib::string & rankProfile) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp index 6c8ef4c30d1..3d6fd229c7b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".minimal_document_retriever"); - #include "minimal_document_retriever.h" #include <vespa/searchlib/docstore/idocumentstore.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp b/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp index 7e382ca5087..bdf2dd486ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/ooscli.cpp @@ -4,6 +4,7 @@ #include "proton.h" #include <vespa/slobrok/sbmirror.h> #include <vespa/messagebus/network/oosmanager.h> +#include <vespa/fnet/frt/supervisor.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.ooscli"); @@ -49,5 +50,4 @@ OosCli::PerformTask() Schedule(1.0); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp index 215f21b66a8..068575ae96c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/operationdonecontext.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "operationdonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> -namespace proton -{ - +namespace proton { OperationDoneContext::OperationDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, @@ -17,13 +14,11 @@ OperationDoneContext::OperationDoneContext(std::unique_ptr<FeedToken> token, { } - OperationDoneContext::~OperationDoneContext() { ack(); } - void OperationDoneContext::ack() { @@ -33,12 +28,10 @@ OperationDoneContext::ack() } } - bool OperationDoneContext::shouldTrace(uint32_t traceLevel) { return _token ? _token->shouldTrace(traceLevel) : false; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp index 07a37a1b299..24f7fd4f19c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "persistencehandlerproxy.h" #include "documentretriever.h" -#include <vespa/persistence/spi/result.h> #include "documentdb.h" #include <vespa/searchcore/proton/feedoperation/createbucketoperation.h> #include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h> @@ -12,8 +10,7 @@ #include <vespa/searchcore/proton/feedoperation/removeoperation.h> #include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h> #include <vespa/searchcore/proton/feedoperation/updateoperation.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.persistencehandlerproxy"); +#include <vespa/persistence/spi/result.h> using storage::spi::Bucket; using storage::spi::Timestamp; @@ -29,20 +26,17 @@ PersistenceHandlerProxy::PersistenceHandlerProxy(const DocumentDB::SP &documentD _documentDB->retain(); } - PersistenceHandlerProxy::~PersistenceHandlerProxy() { _documentDB->release(); } - void PersistenceHandlerProxy::initialize() { _documentDB->waitForOnlineState(); } - void PersistenceHandlerProxy::handlePut(FeedToken token, const Bucket &bucket, @@ -54,7 +48,6 @@ PersistenceHandlerProxy::handlePut(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleUpdate(FeedToken token, const Bucket &bucket, @@ -67,7 +60,6 @@ PersistenceHandlerProxy::handleUpdate(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleRemove(FeedToken token, const Bucket &bucket, @@ -80,14 +72,12 @@ PersistenceHandlerProxy::handleRemove(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleListBuckets(IBucketIdListResultHandler &resultHandler) { _bucketHandler.handleListBuckets(resultHandler); } - void PersistenceHandlerProxy::handleSetClusterState(const storage::spi::ClusterState &calc, IGenericResultHandler &resultHandler) @@ -95,7 +85,6 @@ PersistenceHandlerProxy::handleSetClusterState(const storage::spi::ClusterState _clusterStateHandler.handleSetClusterState(calc, resultHandler); } - void PersistenceHandlerProxy::handleSetActiveState( const storage::spi::Bucket &bucket, @@ -106,7 +95,6 @@ PersistenceHandlerProxy::handleSetActiveState( newState, resultHandler); } - void PersistenceHandlerProxy::handleGetBucketInfo(const Bucket &bucket, IBucketInfoResultHandler &resultHandler) @@ -114,7 +102,6 @@ PersistenceHandlerProxy::handleGetBucketInfo(const Bucket &bucket, _bucketHandler.handleGetBucketInfo(bucket, resultHandler); } - void PersistenceHandlerProxy::handleCreateBucket(FeedToken token, const Bucket &bucket) @@ -124,7 +111,6 @@ PersistenceHandlerProxy::handleCreateBucket(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleDeleteBucket(FeedToken token, const Bucket &bucket) @@ -134,14 +120,12 @@ PersistenceHandlerProxy::handleDeleteBucket(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleGetModifiedBuckets(IBucketIdListResultHandler &resultHandler) { _clusterStateHandler.handleGetModifiedBuckets(resultHandler); } - void PersistenceHandlerProxy::handleSplit(FeedToken token, const Bucket &source, @@ -157,7 +141,6 @@ PersistenceHandlerProxy::handleSplit(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - void PersistenceHandlerProxy::handleJoin(FeedToken token, const Bucket &source1, @@ -173,7 +156,6 @@ PersistenceHandlerProxy::handleJoin(FeedToken token, _feedHandler.handleOperation(token, std::move(op)); } - IPersistenceHandler::RetrieversSP PersistenceHandlerProxy::getDocumentRetrievers(storage::spi::ReadConsistency consistency) { @@ -186,7 +168,6 @@ PersistenceHandlerProxy::lockBucket(const storage::spi::Bucket &bucket) return _documentDB->lockBucket(bucket.getBucketId().stripUnused()); } - void PersistenceHandlerProxy::handleListActiveBuckets( IBucketIdListResultHandler &resultHandler) @@ -194,7 +175,6 @@ PersistenceHandlerProxy::handleListActiveBuckets( _bucketHandler.handleListActiveBuckets(resultHandler); } - void PersistenceHandlerProxy::handlePopulateActiveBuckets( document::BucketId::List &buckets, @@ -203,5 +183,4 @@ PersistenceHandlerProxy::handlePopulateActiveBuckets( _bucketHandler.handlePopulateActiveBuckets(buckets, resultHandler); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp index 22798376055..a8a9cbfead2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistenceproviderproxy.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "persistenceproviderproxy.h" using storage::spi::PersistenceProvider; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ae9aa130b72..271c127bcee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "data_directory_upgrader.h" #include "disk_mem_usage_sampler.h" #include "document_db_explorer.h" #include "flushhandlerproxy.h" #include "memoryflush.h" #include "persistencehandlerproxy.h" -#include "persistenceproviderproxy.h" #include "proton.h" #include "resource_usage_explorer.h" #include "searchhandlerproxy.h" @@ -14,28 +12,24 @@ #include "proton_config_snapshot.h" #include "fileconfigmanager.h" -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/messagebus/emptyreply.h> -#include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/hw_info_sampler.h> +#include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h> #include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h> #include <vespa/searchcore/proton/flushengine/tls_stats_factory.h> -#include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> -#include <vespa/searchlib/aggregation/forcelink.hpp> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/expression/forcelink.hpp> #include <vespa/searchlib/transactionlog/trans_log_server_explorer.h> #include <vespa/searchlib/util/fileheadertk.h> -#include <vespa/vespalib/data/fileheader.h> -#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/document/base/exceptions.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/random.h> -#include <vespa/searchcore/proton/common/hw_info_sampler.h> -#include <vespa/searchcore/proton/reference/document_db_reference_registry.h> -#include <vespa/searchcore/proton/reference/i_document_db_reference.h> -#include <vespa/document/base/exceptions.h> + +#include <vespa/searchlib/aggregation/forcelink.hpp> +#include <vespa/searchlib/expression/forcelink.hpp> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.proton"); @@ -54,7 +48,6 @@ using search::transactionlog::DomainStats; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::internal::InternalProtonType; using document::CompressionConfig; -using searchcorespi::IIndexManagerFactory; namespace proton { @@ -158,7 +151,6 @@ Proton::Proton(const config::ConfigUri & configUri, search::engine::MonitorServer(), IDocumentDBOwner(), StatusProducer(), - PersistenceProviderFactory(), IPersistenceEngineOwner(), ComponentConfigProducer(), _configUri(configUri), @@ -169,7 +161,6 @@ Proton::Proton(const config::ConfigUri & configUri, _tls(), _diskMemUsageSampler(), _persistenceEngine(), - _persistenceProxy(), _documentDBMap(), _matchEngine(), _summaryEngine(), @@ -178,7 +169,6 @@ Proton::Proton(const config::ConfigUri & configUri, _flushEngine(), _rpcHooks(), _healthAdapter(*this), - _componentConfig(), _genericStateHandler(CUSTOM_COMPONENT_API_PATH, *this), _customComponentBindToken(), _customComponentRootToken(), @@ -194,8 +184,6 @@ Proton::Proton(const config::ConfigUri & configUri, _queryLimiter(), _clock(0.010), _threadPool(128 * 1024), - _libraries(), - _indexManagerFactoryRegistry(), _configGenMonitor(), _configGen(0), _distributionKey(-1), @@ -226,11 +214,6 @@ Proton::init() auto bootstrapConfig = configSnapshot->getBootstrapConfig(); assert(bootstrapConfig); - const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); - - if (!performDataDirectoryUpgrade(protonConfig.basedir)) { - _abortInit = true; - } return bootstrapConfig; } @@ -251,10 +234,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) (protonConfig.basedir, diskMemUsageSamplerConfig(protonConfig)); - _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton", - configSnapshot->getGeneration(), - "config obtained at startup")); - _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); _fileHeaderContext.setClusterName(protonConfig.clustername, @@ -303,7 +282,10 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024)); - _summaryExecutor.reset(new vespalib::ThreadStackExecutor(protonConfig.summary.log.numthreads, 128*1024)); + // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, + // but we will not be cheap and give #documentsdbs * 2 + const size_t summaryThreads = std::max(size_t(protonConfig.summary.log.numthreads), protonConfig.documentdb.size() * 2); + _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16)); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor> @@ -313,16 +295,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _protonConfigurer.applyInitialConfig(initializeThreads); initializeThreads.reset(); - if (_persistenceEngine.get() != NULL) { - _persistenceProxy.reset(new ProviderStub(protonConfig. - persistenceprovider.port, - protonConfig. - persistenceprovider.threads, - *configSnapshot-> - getDocumentTypeRepoSP(), - *this)); - } - RPCHooks::Params rpcParams(*this, protonConfig.rpcport, _configUri.getConfigId()); rpcParams.slobrok_config = _configUri.createWithNewId(protonConfig.slobrokconfigid); _rpcHooks.reset(new RPCHooks(rpcParams)); @@ -337,78 +309,25 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _executor.sync(); waitForOnlineState(); _isReplayDone = true; - bool startOk = _fs4Server->start(); + if ( ! _fs4Server->start() ) { + throw vespalib::PortListenException(protonConfig.ptport, "FS4"); + } int port = _fs4Server->getListenPort(); _matchEngine->setOnline(); _matchEngine->setInService(); - LOG(debug, - "Started fs4 interface (startOk=%s, port=%d)", - startOk ? "true" : "false", - port); + LOG(debug, "Started fs4 interface on port %d", port); _flushEngine->start(); _isInitializing = false; _protonConfigurer.setAllowReconfig(true); _initComplete = true; } -bool -Proton::performDataDirectoryUpgrade(const vespalib::string &baseDir) -{ - // TODO: Remove this functionality when going to Vespa 6. - vespalib::string scanDir = baseDir.substr(0, baseDir.rfind('/')); - LOG(debug, "About to perform data directory upgrade: scanDir='%s', destDir='%s'", - scanDir.c_str(), baseDir.c_str()); - DataDirectoryUpgrader upgrader(scanDir, baseDir); - DataDirectoryUpgrader::ScanResult scanResult = upgrader.scan(); - DataDirectoryUpgrader::UpgradeResult upgradeResult = upgrader.upgrade(scanResult); - if (upgradeResult.getStatus() == DataDirectoryUpgrader::ERROR) { - LOG(error, "Data directory upgrade failed: '%s'. Please consult Vespa release notes on how to manually fix this issue. " - "The search node will not start until this issue has been fixed", upgradeResult.getDesc().c_str()); - return false; - } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::IGNORE) { - LOG(debug, "Data directory upgrade ignored: %s", upgradeResult.getDesc().c_str()); - } else if (upgradeResult.getStatus() == DataDirectoryUpgrader::COMPLETE) { - LOG(info, "Data directory upgrade completed: %s", upgradeResult.getDesc().c_str()); - } - return true; -} - -void -Proton::loadLibrary(const vespalib::string &libName) -{ - searchcorespi::IIndexManagerFactory::SP factory(_libraries.create(libName)); - if (factory.get() != NULL) { - LOG(info, "Successfully created index manager factory from library '%s'", libName.c_str()); - _indexManagerFactoryRegistry.add(libName, factory); - } else { - LOG(error, "Failed creating index manager factory from library '%s'", libName.c_str()); - } -} - -searchcorespi::IIndexManagerFactory::SP -Proton::getIndexManagerFactory(const vespalib::stringref & name) const -{ - return _indexManagerFactoryRegistry.get(name); -} - BootstrapConfig::SP Proton::getActiveConfigSnapshot() const { return _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig(); } -storage::spi::PersistenceProvider::UP -Proton::create() const -{ - //TODO : Might be an idea to grab a lock here as this is not - //controlled by you. Must lock with add/remove documentdb or - //reconfig or whatever. - if (_persistenceEngine.get() == NULL) - return storage::spi::PersistenceProvider::UP(); - return storage::spi::PersistenceProvider:: - UP(new PersistenceProviderProxy(*_persistenceEngine)); -} - void Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) { @@ -420,15 +339,8 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.mincoverage, protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - // XXX: This assumes no feeding during reconfig. Otherwise queued messages - // might incorrectly use freed document type repo. - if (_persistenceProxy.get() != NULL) { - _persistenceProxy->setRepo(*repo); - } - _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("proton.documentdbs", - configSnapshot->getGeneration())); - _diskMemUsageSampler-> - setConfig(diskMemUsageSamplerConfig(protonConfig)); + + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig)); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); @@ -446,11 +358,8 @@ Proton::addDocumentDB(const DocTypeName & docTypeName, const DocumentTypeRepo::SP repo = bootstrapConfig->getDocumentTypeRepoSP(); const document::DocumentType *docType = repo->getDocumentType(docTypeName.getName()); if (docType != NULL) { - LOG(info, - "Add document database: " - "doctypename(%s), configid(%s)", - docTypeName.toString().c_str(), - configId.c_str()); + LOG(info, "Add document database: doctypename(%s), configid(%s)", + docTypeName.toString().c_str(), configId.c_str()); return addDocumentDB(*docType, bootstrapConfig, documentDBConfig, initializeThreads).get(); } else { @@ -516,7 +425,6 @@ Proton::~Proton() if (_fs4Server) { _fs4Server->shutDown(); } - _persistenceProxy.reset(); while (!_documentDBMap.empty()) { const DocTypeName docTypeName(_documentDBMap.begin()->first); removeDocumentDB(docTypeName); @@ -552,7 +460,7 @@ size_t Proton::getNumActiveDocs() const vespalib::string -Proton::getDelayedConfigs(void) const +Proton::getDelayedConfigs() const { std::ostringstream res; bool first = true; @@ -773,45 +681,6 @@ Proton::prepareRestart() return true; } -void -Proton::listDocTypes(std::vector<vespalib::string> &documentTypes) -{ - DocumentDBMap dbs; - { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); - dbs = _documentDBMap; - } - for (const auto &kv : dbs) { - vespalib::string documentType; - const DocTypeName &docTypeName = - kv.second->getDocTypeName(); - documentTypes.push_back(docTypeName.getName()); - } -} - - -void -Proton::listSchema(const vespalib::string &documentType, - std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations) -{ - DocumentDB::SP ddb; - DocTypeName docTypeName(documentType); - { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); - DocumentDBMap::const_iterator it = _documentDBMap.find(docTypeName); - if (it != _documentDBMap.end()) - ddb = it->second; - } - if (ddb.get() == NULL) - return; - ddb->listSchema(fieldNames, fieldDataTypes, fieldCollectionTypes, - fieldLocations); -} - - namespace { int countOpenFiles() @@ -883,7 +752,7 @@ Proton::waitForOnlineState() void Proton::getComponentConfig(Consumer &consumer) { - _componentConfig.getComponentConfig(consumer); + _protonConfigurer.getComponentConfig().getComponentConfig(consumer); std::vector<DocumentDB::SP> dbs; { std::shared_lock<std::shared_timed_mutex> guard(_mutex); @@ -904,21 +773,9 @@ Proton::getComponentConfig(Consumer &consumer) } int64_t -Proton::getConfigGeneration(void) +Proton::getConfigGeneration() { - int64_t g = _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getGeneration(); - std::vector<DocumentDB::SP> dbs; - { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); - for (const auto &kv : _documentDBMap) { - dbs.push_back(kv.second); - } - } - for (const auto &docDb : dbs) { - int64_t ddbActiveGen = docDb->getActiveGeneration(); - g = std::min(g, ddbActiveGen); - } - return g; + return _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getGeneration(); } @@ -1012,5 +869,4 @@ Proton::getDocumentDBReferenceRegistry() const return _documentDBReferenceRegistry; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 68f6f4efc6a..5c4181ab5b6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,7 +12,6 @@ #include "proton_configurer.h" #include "rpc_hooks.h" #include "bootstrapconfig.h" -#include <vespa/persistence/proxy/providerstub.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> @@ -23,8 +22,6 @@ #include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> #include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h> -#include <vespa/searchcorespi/plugin/factoryloader.h> -#include <vespa/searchcorespi/plugin/factoryregistry.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/engine/monitorapi.h> #include <vespa/searchlib/engine/transportserver.h> @@ -32,7 +29,6 @@ #include <vespa/vespalib/net/component_config_producer.h> #include <vespa/vespalib/net/generic_state_handler.h> #include <vespa/vespalib/net/json_get_handler.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/net/state_server.h> #include <vespa/vespalib/util/varholder.h> @@ -49,7 +45,6 @@ class Proton : public IProtonConfigurerOwner, public search::engine::MonitorServer, public IDocumentDBOwner, public StatusProducer, - public storage::spi::ProviderStub::PersistenceProviderFactory, public IPersistenceEngineOwner, public vespalib::ComponentConfigProducer, public vespalib::StateExplorer @@ -60,7 +55,6 @@ private: typedef search::engine::MonitorRequest MonitorRequest; typedef search::engine::MonitorReply MonitorReply; typedef search::engine::MonitorClient MonitorClient; - typedef storage::spi::ProviderStub ProviderStub; typedef std::map<DocTypeName, DocumentDB::SP> DocumentDBMap; typedef BootstrapConfig::ProtonConfigSP ProtonConfigSP; typedef std::shared_ptr<FastOS_DynamicLibrary> DynamicLibrarySP; @@ -109,8 +103,7 @@ private: TLS::UP _tls; std::unique_ptr<DiskMemUsageSampler> _diskMemUsageSampler; PersistenceEngine::UP _persistenceEngine; - ProviderStub::UP _persistenceProxy; - DocumentDBMap _documentDBMap; + DocumentDBMap _documentDBMap; MatchEngine::UP _matchEngine; SummaryEngine::UP _summaryEngine; DocsumBySlime::UP _docsumBySlime; @@ -118,7 +111,6 @@ private: FlushEngine::UP _flushEngine; RPCHooks::UP _rpcHooks; HealthAdapter _healthAdapter; - vespalib::SimpleComponentConfigProducer _componentConfig; vespalib::GenericStateHandler _genericStateHandler; vespalib::JsonHandlerRepo::Token::UP _customComponentBindToken; vespalib::JsonHandlerRepo::Token::UP _customComponentRootToken; @@ -132,8 +124,6 @@ private: matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; FastOS_ThreadPool _threadPool; - searchcorespi::FactoryLoader _libraries; - searchcorespi::FactoryRegistry _indexManagerFactoryRegistry; vespalib::Monitor _configGenMonitor; int64_t _configGen; uint32_t _distributionKey; @@ -147,9 +137,6 @@ private: std::unique_ptr<HwInfoSampler> _hwInfoSampler; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; - bool performDataDirectoryUpgrade(const vespalib::string &baseDir); - void loadLibrary(const vespalib::string &libName); - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName & docTypeName, const vespalib::string & configid, const BootstrapConfig::SP & bootstrapConfig, @@ -170,9 +157,6 @@ private: void waitForInitDone(); void waitForOnlineState(); - virtual storage::spi::PersistenceProvider::UP create() const override; - searchcorespi::IIndexManagerFactory::SP - getIndexManagerFactory(const vespalib::stringref & name) const override; uint32_t getDistributionKey() const override { return _distributionKey; } BootstrapConfig::SP getActiveConfigSnapshot() const; virtual std::shared_ptr<IDocumentDBReferenceRegistry> getDocumentDBReferenceRegistry() const override; @@ -194,18 +178,15 @@ public: * * 1st phase init: start cheap clock thread and get initial config */ - BootstrapConfig::SP - init(); + BootstrapConfig::SP init(); /* * 2nd phase init: setup data structures. */ - void - init(const BootstrapConfig::SP & configSnapshot); + void init(const BootstrapConfig::SP & configSnapshot); - DocumentDB::SP - getDocumentDB(const document::DocumentType &docType); + DocumentDB::SP getDocumentDB(const document::DocumentType &docType); DocumentDB::SP addDocumentDB(const document::DocumentType &docType, @@ -218,15 +199,6 @@ public: bool triggerFlush(); bool prepareRestart(); - void listDocTypes(std::vector<vespalib::string> &documentTypes); - - void - listSchema(const vespalib::string &documentType, - std::vector<vespalib::string> &fieldNames, - std::vector<vespalib::string> &fieldDataTypes, - std::vector<vespalib::string> &fieldCollectionTypes, - std::vector<vespalib::string> &fieldLocations); - // implements ComponentConfigProducer interface virtual void getComponentConfig(Consumer &consumer) override; @@ -237,18 +209,17 @@ public: /** * Return the oldest active config generation used by proton. */ - int64_t getConfigGeneration(void); + int64_t getConfigGeneration(); size_t getNumDocs() const; size_t getNumActiveDocs() const; DocsumBySlime & getDocsumBySlime() { return *_docsumBySlime; } - vespalib::string getDelayedConfigs(void) const; + vespalib::string getDelayedConfigs() const; virtual StatusReport::List getStatusReports() const override; MatchEngine & getMatchEngine() { return *_matchEngine; } - FlushEngine & getFlushEngine() { return *_flushEngine; } vespalib::ThreadStackExecutorBase & getExecutor() { return _executor; } bool isReplayDone() const { return _isReplayDone; } @@ -267,4 +238,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index 9bfc8798f44..b5d76c3f60a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -7,7 +7,7 @@ #include "proton_config_snapshot.h" #include "i_proton_configurer.h" #include <vespa/log/log.h> -LOG_SETUP(".proton.server.protonconfigurer"); +LOG_SETUP(".proton.server.proton_config_fetcher"); using namespace vespa::config::search; using namespace vespa::config::search::core; @@ -87,17 +87,21 @@ ProtonConfigFetcher::updateDocumentDBConfigs(const BootstrapConfig::SP & bootstr void ProtonConfigFetcher::reconfigure() { + auto bootstrapConfig = _bootstrapConfigManager.getConfig(); + int64_t generation = bootstrapConfig->getGeneration(); std::map<DocTypeName, DocumentDBConfig::SP> dbConfigs; { lock_guard guard(_mutex); for (auto &kv : _dbManagerMap) { auto insres = dbConfigs.insert(std::make_pair(kv.first, kv.second->getConfig())); assert(insres.second); + assert(insres.first->second->getGeneration() == generation); } } - auto bootstrapConfig = _bootstrapConfigManager.getConfig(); auto configSnapshot = std::make_shared<ProtonConfigSnapshot>(std::move(bootstrapConfig), std::move(dbConfigs)); + LOG(debug, "Reconfiguring proton with gen %" PRId64, generation); _owner.reconfigure(std::move(configSnapshot)); + LOG(debug, "Reconfigured proton with gen %" PRId64, generation); } void @@ -159,7 +163,7 @@ ProtonConfigFetcher::start() fetchConfigs(); if (_threadPool.NewThread(this, NULL) == NULL) { throw vespalib::IllegalStateException( - "Failed starting thread for proton configurer"); + "Failed starting thread for proton config fetcher"); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 6c859a0db5d..ae8996b2cb2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -32,12 +32,12 @@ public: int64_t getGeneration() const; /** - * Start configurer, callbacks may come from now on. + * Start config fetcher, callbacks may come from now on. */ void start(); /** - * Shutdown configurer, ensuring that no more callbacks arrive + * Shutdown config fetcher, ensuring that no more callbacks arrive */ void close(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index c7009754899..0d2f6e6c867 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -23,7 +23,8 @@ ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, _pendingConfigSnapshot(), _activeConfigSnapshot(), _mutex(), - _allowReconfig(false) + _allowReconfig(false), + _componentConfig() { } @@ -116,6 +117,8 @@ ProtonConfigurer::applyConfig(std::shared_ptr<ProtonConfigSnapshot> configSnapsh configureDocumentDB(*configSnapshot, docTypeName, ddbConfig.configid, initializeThreads); } pruneDocumentDBs(*configSnapshot); + size_t gen = bootstrapConfig->getGeneration(); + _componentConfig.addConfig({"proton", gen}); std::lock_guard<std::mutex> guard(_mutex); _activeConfigSnapshot = configSnapshot; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index 17f25274876..b8c80602691 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -4,6 +4,7 @@ #include "i_proton_configurer.h" #include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/vespalib/net/simple_component_config_producer.h> #include <map> #include <mutex> #include "executor_thread_service.h" @@ -30,6 +31,7 @@ class ProtonConfigurer : public IProtonConfigurer std::shared_ptr<ProtonConfigSnapshot> _activeConfigSnapshot; mutable std::mutex _mutex; bool _allowReconfig; + vespalib::SimpleComponentConfigProducer _componentConfig; void performReconfigure(); bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig); @@ -52,6 +54,7 @@ public: virtual void reconfigure(std::shared_ptr<ProtonConfigSnapshot> configSnapshot) override; void applyInitialConfig(InitializeThreads initializeThreads); + vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp index 5282d861cb6..7941803086a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/prune_session_cache_job.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "prune_session_cache_job.h" #include <vespa/fastos/timestamp.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp index 6d67a989df1..d88414c11c2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.pruneremoveddocumentsjob"); #include "pruneremoveddocumentsjob.h" -#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> -#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> #include "ipruneremoveddocumentshandler.h" #include "ifrozenbuckethandler.h" +#include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> +#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.pruneremoveddocumentsjob"); using document::BucketId; using storage::spi::Timestamp; @@ -65,7 +65,7 @@ PruneRemovedDocumentsJob::flush(DocId lowLid, DocId nextLowLid, bool -PruneRemovedDocumentsJob::run(void) +PruneRemovedDocumentsJob::run() { uint64_t tshz = 1000000; fastos::TimeStamp now = fastos::ClockSystem::now(); diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index b157b652f24..4fda9f43881 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "putdonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/common/docid_limit.h> -namespace proton -{ - +namespace proton { PutDoneContext::PutDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, @@ -18,7 +15,6 @@ PutDoneContext::PutDoneContext(std::unique_ptr<FeedToken> token, { } - PutDoneContext::~PutDoneContext() { if (_docIdLimit != nullptr) { @@ -26,5 +22,4 @@ PutDoneContext::~PutDoneContext() } } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 2ec8787a8ba..bccb8fe64ba 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.reconfig_params"); #include "reconfig_params.h" namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index 5baa903b0d9..367e03d0f41 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "removedonecontext.h" #include "removedonetask.h" #include <vespa/searchcore/proton/common/feedtoken.h> -namespace proton -{ - +namespace proton { RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, @@ -24,7 +21,6 @@ RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, } } - RemoveDoneContext::~RemoveDoneContext() { ack(); @@ -34,5 +30,4 @@ RemoveDoneContext::~RemoveDoneContext() } } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp index 423af443165..bdcc679fc2f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonetask.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "removedonetask.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> -namespace proton -{ - +namespace proton { RemoveDoneTask::RemoveDoneTask(IDocumentMetaStore &documentMetaStore, uint32_t lid) @@ -16,12 +13,10 @@ RemoveDoneTask::RemoveDoneTask(IDocumentMetaStore &documentMetaStore, { } - RemoveDoneTask::~RemoveDoneTask() { } - void RemoveDoneTask::run() { @@ -30,5 +25,4 @@ RemoveDoneTask::run() } } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 81e607a53a3..dc0213206e2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "resource_usage_explorer.h" #include "disk_mem_usage_filter.h" #include <vespa/vespalib/data/slime/cursor.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 58c2fd90625..1d255c388d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.rtchooks"); - #include "rpc_hooks.h" #include "proton.h" -#include <vespa/messagebus/emptyreply.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/closuretask.h> -#include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/fnet/frt/supervisor.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.rtchooks"); using namespace vespalib; using document::CompressionConfig; @@ -121,12 +118,12 @@ RPCHooksBase::Session::Session() void RPCHooksBase::initRPC() { - _orb.SetSessionInitHook(FRT_METHOD(RPCHooksBase::initSession), this); - _orb.SetSessionFiniHook(FRT_METHOD(RPCHooksBase::finiSession), this); - _orb.SetSessionDownHook(FRT_METHOD(RPCHooksBase::downSession), this); - _orb.SetMethodMismatchHook(FRT_METHOD(RPCHooksBase::mismatch), this); + _orb->SetSessionInitHook(FRT_METHOD(RPCHooksBase::initSession), this); + _orb->SetSessionFiniHook(FRT_METHOD(RPCHooksBase::finiSession), this); + _orb->SetSessionDownHook(FRT_METHOD(RPCHooksBase::downSession), this); + _orb->SetMethodMismatchHook(FRT_METHOD(RPCHooksBase::mismatch), this); - FRT_ReflectionBuilder rb(&_orb); + FRT_ReflectionBuilder rb(_orb.get()); //------------------------------------------------------------------------- rb.DefineMethod("proton.enableSearching", "", "", true, FRT_METHOD(RPCHooksBase::rpc_enableSearching), this); @@ -181,30 +178,6 @@ RPCHooksBase::initRPC() "such that TLS replay time + time spent flushing components is as low as possible"); rb.ReturnDesc("success", "Whether or not prepare for restart was triggered."); //------------------------------------------------------------------------- - rb.DefineMethod("proton.listDocTypes", "", "S", true, - FRT_METHOD(RPCHooksBase::rpc_listDocTypes), this); - rb.MethodDesc("Get the current list of document types"); - rb.ReturnDesc("documentTypes", "Current list of document types"); - //------------------------------------------------------------------------- - rb.DefineMethod("proton.listSchema", "s", "SSSS", true, - FRT_METHOD(RPCHooksBase::rpc_listSchema), this); - rb.MethodDesc("Get the current schema for given document type"); - rb.ParamDesc("documentType", "Document type name"); - rb.ReturnDesc("fieldNames", "Schema field names"); - rb.ReturnDesc("fieldDataTypes", "Schema field data types"); - rb.ReturnDesc("fieldCollTypes", "Schema field collection types"); - rb.ReturnDesc("fieldLocation", "Schema field locations"); - //------------------------------------------------------------------------- - rb.DefineMethod("proton.getConfigTime", "", "l", true, - FRT_METHOD(RPCHooksBase::rpc_getConfigGeneration), this); - rb.MethodDesc("Get the oldest active config generation"); - rb.ReturnDesc("configTime", "Oldest active config generation"); - //------------------------------------------------------------------------- - rb.DefineMethod("proton.getConfigGeneration", "", "l", true, - FRT_METHOD(RPCHooksBase::rpc_getConfigGeneration), this); - rb.MethodDesc("Get the oldest active config generation"); - rb.ReturnDesc("configTime", "Oldest active config generation"); - //------------------------------------------------------------------------- rb.DefineMethod("proton.getDocsums", "bix", "bix", true, FRT_METHOD(RPCHooksBase::rpc_getDocSums), this); rb.MethodDesc("Get list of document summaries"); rb.ParamDesc("encoding", "0=raw, 6=lz4"); @@ -219,10 +192,10 @@ RPCHooksBase::initRPC() RPCHooksBase::RPCHooksBase(Params ¶ms) : _proton(params.proton), _docsumByRPC(new DocsumByRPC(_proton.getDocsumBySlime())), - _orb(), - _regAPI(_orb, params.slobrok_config), + _orb(std::make_unique<FRT_Supervisor>()), + _regAPI(*_orb, params.slobrok_config), _executor(48, 128 * 1024), - _ooscli(params, _orb) + _ooscli(params, *_orb) { } @@ -231,8 +204,8 @@ RPCHooksBase::open(Params & params) { initRPC(); _regAPI.registerName((params.identity + "/realtimecontroller").c_str()); - _orb.Listen(params.rtcPort); - _orb.Start(); + _orb->Listen(params.rtcPort); + _orb->Start(); LOG(debug, "started monitoring interface"); } @@ -244,7 +217,7 @@ void RPCHooksBase::close() { LOG(info, "shutting down monitoring interface"); - _orb.ShutDown(true); + _orb->ShutDown(true); _executor.shutdown(); { MonitorGuard guard(_stateMonitor); @@ -443,96 +416,6 @@ RPCHooksBase::rpc_prepareRestart(FRT_RPCRequest *req) } void -RPCHooksBase::listDocTypes(FRT_RPCRequest *req) -{ - std::vector<string> documentTypes; - - _proton.listDocTypes(documentTypes); - - FRT_Values &ret = *req->GetReturn(); - FRT_StringValue *dt = ret.AddStringArray(documentTypes.size()); - for (uint32_t i = 0; i < documentTypes.size(); ++i) - ret.SetString(&dt[i], documentTypes[i].c_str()); - - LOG(info, - "RPCHooksBase::listDocTypes finished successfully"); - req->Return(); -} - - -void -RPCHooksBase::rpc_listDocTypes(FRT_RPCRequest *req) -{ - LOG(info, - "RPCHooksBase::rpc_listDocTypes started"); - req->Detach(); - letProtonDo(makeClosure(this, &RPCHooksBase::listDocTypes, req)); -} - - -void -RPCHooksBase::listSchema(FRT_RPCRequest *req) -{ - std::vector<string> fieldNames; - std::vector<string> fieldDataTypes; - std::vector<string> fieldCollectionTypes; - std::vector<string> fieldLocations; - - FRT_Values &arg = *req->GetParams(); - string documentType(arg[0]._string._str, arg[0]._string._len); - - _proton.listSchema(documentType, - fieldNames, fieldDataTypes, fieldCollectionTypes, - fieldLocations); - - FRT_Values &ret = *req->GetReturn(); - FRT_StringValue *fn = ret.AddStringArray(fieldNames.size()); - for (uint32_t i = 0; i < fieldNames.size(); ++i) - ret.SetString(&fn[i], fieldNames[i].c_str()); - - FRT_StringValue *fdt = ret.AddStringArray(fieldDataTypes.size()); - for (uint32_t i = 0; i < fieldDataTypes.size(); ++i) - ret.SetString(&fdt[i], fieldDataTypes[i].c_str()); - - FRT_StringValue *fct = ret.AddStringArray(fieldCollectionTypes.size()); - for (uint32_t i = 0; i < fieldCollectionTypes.size(); ++i) - ret.SetString(&fct[i], fieldCollectionTypes[i].c_str()); - - FRT_StringValue *fl = ret.AddStringArray(fieldLocations.size()); - for (uint32_t i = 0; i < fieldLocations.size(); ++i) - ret.SetString(&fl[i], fieldLocations[i].c_str()); - - LOG(info, - "RPCHooksBase::listSchema finished successfully"); - req->Return(); -} - - -void -RPCHooksBase::rpc_listSchema(FRT_RPCRequest *req) -{ - LOG(info, - "RPCHooksBase::rpc_listSchema started"); - req->Detach(); - letProtonDo(makeClosure(this, &RPCHooksBase::listSchema, req)); -} - - -void -RPCHooksBase::getConfigGeneration(FRT_RPCRequest *req) -{ - int64_t configGeneration = _proton.getConfigGeneration(); - FRT_Values &ret = *req->GetReturn(); - ret.AddInt64(configGeneration); - - LOG(info, - "RPCHooksBase::getConfigGeneration finished successfully, " - "configGeneration=%" PRId64, - configGeneration); - req->Return(); -} - -void RPCHooksBase::rpc_getDocSums(FRT_RPCRequest *req) { LOG(debug, "proton.getDocsums()"); @@ -547,16 +430,6 @@ RPCHooksBase::getDocsums(FRT_RPCRequest *req) req->Return(); } -void -RPCHooksBase::rpc_getConfigGeneration(FRT_RPCRequest *req) -{ - LOG(info, - "RPCHooksBase::rpc_getConfigGeneration started"); - req->Detach(); - letProtonDo(makeClosure(this, &RPCHooksBase::getConfigGeneration, req)); -} - - const RPCHooksBase::Session::SP & RPCHooksBase::getSession(FRT_RPCRequest *req) { diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h index ba0ddaea310..26c7885cadf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h @@ -2,7 +2,6 @@ #pragma once -#include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/sbregister.h> #include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/executor.h> @@ -35,12 +34,12 @@ private: fastos::TimeStamp getCreateTime() const { return _createTime; } Session & setGen(int64_t gen) { _gen = gen; return *this; } - int64_t getNumDocs(void) const { return _numDocs; } + int64_t getNumDocs() const { return _numDocs; } void setNumDocs(int64_t numDocs) { _numDocs = numDocs; } - bool getDown(void) const { return _down; } - void setDown(void) { _down = true; } + bool getDown() const { return _down; } + void setDown() { _down = true; } - const vespalib::string & getDelayedConfigs(void) const { + const vespalib::string & getDelayedConfigs() const { return _delayedConfigs; } @@ -61,13 +60,13 @@ private: fastos::TimeStamp _dueTime; }; - Proton & _proton; - std::unique_ptr<DocsumByRPC> _docsumByRPC; - FRT_Supervisor _orb; - slobrok::api::RegisterAPI _regAPI; - vespalib::Monitor _stateMonitor; - vespalib::ThreadStackExecutor _executor; - OosCli _ooscli; + Proton & _proton; + std::unique_ptr<DocsumByRPC> _docsumByRPC; + std::unique_ptr<FRT_Supervisor> _orb; + slobrok::api::RegisterAPI _regAPI; + vespalib::Monitor _stateMonitor; + vespalib::ThreadStackExecutor _executor; + OosCli _ooscli; void initRPC(); void letProtonDo(vespalib::Closure::UP closure); @@ -79,9 +78,6 @@ private: void checkState(StateArg::UP arg); void reportState(Session & session, FRT_RPCRequest * req) __attribute__((noinline)); void getProtonStatus(FRT_RPCRequest * req); - void listDocTypes(FRT_RPCRequest *req); - void listSchema(FRT_RPCRequest *req); - void getConfigGeneration(FRT_RPCRequest *req); void getDocsums(FRT_RPCRequest *req); static const Session::SP & getSession(FRT_RPCRequest *req); @@ -115,9 +111,6 @@ public: void rpc_die(FRT_RPCRequest *req); void rpc_triggerFlush(FRT_RPCRequest *req); void rpc_prepareRestart(FRT_RPCRequest *req); - void rpc_listDocTypes(FRT_RPCRequest *req); - void rpc_listSchema(FRT_RPCRequest *req); - void rpc_getConfigGeneration(FRT_RPCRequest *req); void rpc_getDocSums(FRT_RPCRequest *req); void initSession(FRT_RPCRequest *req); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 5a485a3744d..77aecb1cb5e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sample_attribute_usage_job.h" #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> @@ -15,8 +14,7 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, double interval) - : IMaintenanceJob("sample_attribute_usage." + docTypeName, - 0.0, interval), + : IMaintenanceJob("sample_attribute_usage." + docTypeName, 0.0, interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), _attributeUsageFilter(attributeUsageFilter) @@ -30,13 +28,11 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> - (_attributeUsageFilter); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter); _readyAttributeManager->asyncForEachAttribute( std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute( - std::make_shared<AttributeUsageSamplerFunctor>(context, - "notready")); + std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp index 99770b0af75..cbba7d5c5a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "searchcontext.h" using search::queryeval::Searchable; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp index 8bb6b29b73a..4eff0fcdfb1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp @@ -1,14 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "searchhandlerproxy.h" #include "documentdb.h" #include <vespa/searchlib/engine/searchreply.h> #include <vespa/searchlib/engine/docsumreply.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.searchhandlerproxy"); - namespace proton { SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB) @@ -17,7 +13,7 @@ SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB) _documentDB->retain(); } -SearchHandlerProxy::~SearchHandlerProxy(void) +SearchHandlerProxy::~SearchHandlerProxy() { _documentDB->release(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp index 9b71c069cb7..8c2e22acc07 100644 --- a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "simpleflush.h" #include <algorithm> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index c09ba707295..d09070f6124 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> @@ -128,6 +129,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _getSerialNum(ctx._getSerialNum), _tlsSyncer(ctx._writeService.master(), ctx._getSerialNum, ctx._tlSyncer), _dmsFlushTarget(), + _dmsShrinkTarget(), _subDbId(cfg._subDbId), _subDbType(cfg._subDbType), _fileHeaderContext(*this, ctx._fileHeaderContext, _docTypeName, _baseDir), @@ -174,7 +176,14 @@ StoreOnlyDocSubDB::hasDocument(const document::DocumentId &id) void StoreOnlyDocSubDB::onReplayDone() { - _metaStoreCtx->get().constructFreeList(); + _dms->constructFreeList(); + _dms->shrinkLidSpace(); + uint32_t docIdLimit = _dms->getCommittedDocIdLimit(); + auto &docStore = _rSummaryMgr->getBackingStore(); + if (docIdLimit < docStore.getDocIdLimit()) { + docStore.compactLidSpace(docIdLimit); + docStore.shrinkLidSpace(); + } } @@ -191,6 +200,7 @@ StoreOnlyDocSubDB::getOldestFlushedSerial() { SerialNum lowest(_iSummaryMgr->getBackingStore().lastSyncToken()); lowest = std::min(lowest, _dmsFlushTarget->getFlushedSerialNum()); + lowest = std::min(lowest, _dmsShrinkTarget->getFlushedSerialNum()); return lowest; } @@ -200,6 +210,7 @@ StoreOnlyDocSubDB::getNewestFlushedSerial() { SerialNum highest(_iSummaryMgr->getBackingStore().lastSyncToken()); highest = std::max(highest, _dmsFlushTarget->getFlushedSerialNum()); + highest = std::max(highest, _dmsShrinkTarget->getFlushedSerialNum()); return highest; } @@ -282,6 +293,14 @@ StoreOnlyDocSubDB::setupDocumentMetaStore(DocumentMetaStoreInitializerResult::SP baseDir, dmsResult->tuneFile(), _fileHeaderContext, _hwInfo)); + using Type = IFlushTarget::Type; + using Component = IFlushTarget::Component; + _dmsShrinkTarget = std::make_shared<ShrinkLidSpaceFlushTarget> + ("documentmetastore.shrink", + Type::GC, Component::ATTRIBUTE, + _flushedDocumentMetaStoreSerialNum, + _dmsFlushTarget->getLastFlushTime(), + dms); } DocumentSubDbInitializer::UP @@ -345,6 +364,7 @@ StoreOnlyDocSubDB::getFlushTargetsInternal() { IFlushTarget::List ret(_rSummaryMgr->getFlushTargets()); ret.push_back(_dmsFlushTarget); + ret.push_back(_dmsShrinkTarget); return ret; } @@ -501,7 +521,7 @@ StoreOnlyDocSubDB::close() { assert(_writeService.master().isCurrentThread()); search::IDocumentStore & store(_rSummaryMgr->getBackingStore()); - SerialNum syncToken = store.initFlush(store.lastSyncToken()); + SerialNum syncToken = store.initFlush(store.tentativeLastSyncToken()); _tlSyncer.sync(syncToken); store.flush(syncToken); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index a6fe412b543..0a388a8343f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -28,6 +28,7 @@ class LegacyDocumentDBMetrics; class FeedHandler; class DocumentMetaStoreInitializerResult; class IDocumentSubDBOwner; +class ShrinkLidSpaceFlushTarget; namespace initializer { class InitializerTask; } namespace bucketdb { class IBucketDBHandlerInitializer; } @@ -166,6 +167,7 @@ private: const IGetSerialNum &_getSerialNum; TlsSyncer _tlsSyncer; DocumentMetaStoreFlushTarget::SP _dmsFlushTarget; + std::shared_ptr<ShrinkLidSpaceFlushTarget> _dmsShrinkTarget; IFlushTargetList getFlushTargets() override; protected: diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index ac24469e4d2..e27a710022f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -2,23 +2,21 @@ #include "ireplayconfig.h" #include "storeonlyfeedview.h" -#include <vespa/searchcore/proton/common/bucketfactory.h> -#include <vespa/searchcore/proton/common/commit_time_tracker.h> -#include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/metrics/feed_metrics.h> -#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/vespalib/util/closuretask.h> -#include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include "forcecommitcontext.h" #include "operationdonecontext.h" #include "removedonecontext.h" #include "updatedonecontext.h" #include "putdonecontext.h" -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/searchcore/proton/common/commit_time_tracker.h> +#include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchcore/proton/metrics/feed_metrics.h> +#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> #include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/util/closuretask.h> + #include <vespa/log/log.h> LOG_SETUP(".proton.server.storeonlyfeedview"); diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp index 73a3d64f58d..dad5316e97b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.summaryadapter"); #include "summaryadapter.h" #include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.summaryadapter"); + using namespace document; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp index f17538836bc..89028e5123b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tlssyncer.h" #include "igetserialnum.h" #include <vespa/vespalib/util/threadexecutor.h> @@ -11,8 +10,7 @@ using search::makeLambdaTask; using search::SerialNum; -namespace proton -{ +namespace proton { TlsSyncer::TlsSyncer(vespalib::ThreadExecutor &executor, const IGetSerialNum &getSerialNum, diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp index 463a5fa9704..667cb597782 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp @@ -1,20 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.transactionlogmanager"); #include "configstore.h" #include "transactionlogmanager.h" #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.transactionlogmanager"); + using vespalib::IllegalStateException; using vespalib::make_string; using search::transactionlog::TransLogClient; -namespace proton -{ +namespace proton { void diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp index 2b7f5ff7944..fec7b95299b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transactionlogmanagerbase.h" -#include <vespa/searchcore/proton/common/eventlogger.h> -#include <stdexcept> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> @@ -81,7 +79,7 @@ TransactionLogManagerBase::internalStartReplay() void -TransactionLogManagerBase::markReplayStarted(void) +TransactionLogManagerBase::markReplayStarted() { vespalib::MonitorGuard guard(_replayMonitor); _replayStarted = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h index 8c1de1fb14f..4d56b912f36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h @@ -68,7 +68,7 @@ public: { return _tlc.getRPCTarget(); } void - markReplayStarted(void); + markReplayStarted(); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp index 82db11aa0f8..964be92a631 100644 --- a/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/updatedonecontext.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "updatedonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 06abf5e65bc..99b39f68006 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -3,6 +3,7 @@ #include <vespa/document/util/compressor.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> +#include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h index 920facf76a2..d811a8924aa 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.h @@ -3,7 +3,8 @@ #include <vespa/searchlib/engine/docsumapi.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/fnet/frt/frt.h> + +class FRT_RPCRequest; namespace proton { @@ -31,4 +32,3 @@ private: }; } - diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index a41f76fef3c..c8df4166261 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".proton.summaryengine.summaryengine"); - #include "summaryengine.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/log.h> +LOG_SETUP(".proton.summaryengine.summaryengine"); + using namespace search::engine; using namespace proton; diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h index b2fc3ff6dbc..f0d10d9ecbd 100644 --- a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h +++ b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h @@ -2,6 +2,7 @@ #pragma once #include "document.h" +#include <cassert> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp index f7b8ce691da..aedc56a83fe 100644 --- a/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/buckethandler.cpp @@ -1,15 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "buckethandler.h" +#include <cassert> - -namespace proton -{ - -namespace test -{ - +namespace proton::test { BucketHandler::BucketHandler() : IBucketStateChangedNotifier(), @@ -17,20 +11,17 @@ BucketHandler::BucketHandler() { } - BucketHandler::~BucketHandler() { assert(_handlers.empty()); } - void BucketHandler::addBucketStateChangedHandler(IBucketStateChangedHandler *handler) { _handlers.insert(handler); } - void BucketHandler::removeBucketStateChangedHandler(IBucketStateChangedHandler * handler) @@ -38,7 +29,6 @@ BucketHandler::removeBucketStateChangedHandler(IBucketStateChangedHandler * _handlers.erase(handler); } - void BucketHandler::notifyBucketStateChanged(const document::BucketId &bucketId, storage::spi::BucketInfo::ActiveState @@ -49,8 +39,4 @@ BucketHandler::notifyBucketStateChanged(const document::BucketId &bucketId, } } - -} // namespace test - -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h index 281c814fc77..4947c7500ed 100644 --- a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h +++ b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h @@ -2,11 +2,12 @@ #pragma once #include <vespa/searchcore/proton/server/ibucketstatecalculator.h> -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucketidlist.h> namespace proton::test { -typedef document::BucketId::List BucketIdVector; +using BucketIdVector = document::bucket::BucketIdList; + typedef std::set<document::BucketId> BucketIdSet; class BucketStateCalculator : public IBucketStateCalculator diff --git a/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp index e241e92806a..20f27e15a87 100644 --- a/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/clusterstatehandler.cpp @@ -1,15 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "clusterstatehandler.h" +#include <cassert> - -namespace proton -{ - -namespace test -{ - +namespace proton::test { ClusterStateHandler::ClusterStateHandler() : IClusterStateChangedNotifier(), @@ -17,13 +11,11 @@ ClusterStateHandler::ClusterStateHandler() { } - ClusterStateHandler::~ClusterStateHandler() { assert(_handlers.empty()); } - void ClusterStateHandler:: addClusterStateChangedHandler(IClusterStateChangedHandler *handler) @@ -31,7 +23,6 @@ addClusterStateChangedHandler(IClusterStateChangedHandler *handler) _handlers.insert(handler); } - void ClusterStateHandler:: removeClusterStateChangedHandler(IClusterStateChangedHandler *handler) @@ -39,7 +30,6 @@ removeClusterStateChangedHandler(IClusterStateChangedHandler *handler) _handlers.erase(handler); } - void ClusterStateHandler:: notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) @@ -49,8 +39,4 @@ notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) } } - -} // namespace test - -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h index b219c3a43be..9bdec51b519 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h @@ -29,7 +29,7 @@ struct DummyDocumentStore : public search::IDocumentStore virtual uint64_t lastSyncToken() const override { return 0; } virtual uint64_t tentativeLastSyncToken() const override { return 0; } virtual fastos::TimeStamp getLastFlushTime() const override { return fastos::TimeStamp(); } - virtual uint64_t nextId() const override { return 0; } + virtual uint32_t getDocIdLimit() const override { return 0; } virtual size_t memoryUsed() const override { return 0; } virtual size_t memoryMeta() const override { return 0; } virtual size_t getDiskFootprint() const override { return 0; } @@ -47,7 +47,7 @@ struct DummyDocumentStore : public search::IDocumentStore virtual double getVisitCost() const override { return 1.0; } virtual search::DataStoreStorageStats getStorageStats() const override { - return search::DataStoreStorageStats(0, 0, 0.0, 0, 0); + return search::DataStoreStorageStats(0, 0, 0.0, 0, 0, 0); } virtual search::MemoryUsage getMemoryUsage() const override { return search::MemoryUsage(); } virtual std::vector<search::DataStoreFileChunkStats> getFileChunkStats() const override { @@ -57,6 +57,7 @@ struct DummyDocumentStore : public search::IDocumentStore virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } virtual bool canShrinkLidSpace() const override { return false; } + virtual size_t getEstimatedShrinkLidSpaceGain() const override { return 0; } virtual void shrinkLidSpace() override {} }; diff --git a/searchcore/src/vespa/searchcore/proton/test/test.h b/searchcore/src/vespa/searchcore/proton/test/test.h index 9aa02f74ace..1c0cd8f7516 100644 --- a/searchcore/src/vespa/searchcore/proton/test/test.h +++ b/searchcore/src/vespa/searchcore/proton/test/test.h @@ -9,7 +9,6 @@ #include "dummy_document_sub_db.h" #include "dummy_feed_view.h" #include "dummy_summary_manager.h" -#include "directory_handler.h" #include "resulthandler.h" #include "userdocuments.h" #include "userdocumentsbuilder.h" diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h index d1ea6f6ab4c..6b5a911352b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h +++ b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h @@ -2,10 +2,9 @@ #pragma once #include "bucketdocuments.h" +#include <map> -namespace proton { - -namespace test { +namespace proton::test { /** * Collection of documents for a set of users, @@ -48,7 +47,4 @@ public: void clear() { _docs.clear(); } }; -} // namespace test - -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/util/CMakeLists.txt b/searchcore/src/vespa/searchcore/util/CMakeLists.txt index 98fce6782ca..fd162c3284c 100644 --- a/searchcore/src/vespa/searchcore/util/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/util/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcore_util STATIC SOURCES - base64encoder.cpp eventloop.cpp log.cpp DEPENDS diff --git a/searchcore/src/vespa/searchcore/util/autoptr.h b/searchcore/src/vespa/searchcore/util/autoptr.h index 9fe58babe3a..e2a3a560b70 100644 --- a/searchcore/src/vespa/searchcore/util/autoptr.h +++ b/searchcore/src/vespa/searchcore/util/autoptr.h @@ -12,23 +12,23 @@ private: FastS_AutoPtr& operator=(const FastS_AutoPtr &); T *_val; - void Clean(void) { + void Clean() { if (_val != NULL) { delete _val; _val = NULL; } } public: - FastS_AutoPtr(void) : _val(NULL) { } + FastS_AutoPtr() : _val(NULL) { } explicit FastS_AutoPtr(T *val) : _val(val) { } - ~FastS_AutoPtr(void) { Clean(); } + ~FastS_AutoPtr() { Clean(); } void Set(T *val) { Clean(); _val = val; } - T *Get(void) const { return _val; } - T *HandOver(void) { T *ret = _val; _val = NULL; return ret; } - void Drop(void) { + T *Get() const { return _val; } + T *HandOver() { T *ret = _val; _val = NULL; return ret; } + void Drop() { if (_val != NULL) { delete _val; _val = NULL; @@ -45,14 +45,14 @@ private: FastS_AutoRefCntPtr& operator=(const FastS_AutoRefCntPtr &); T *_val; - void Clean(void) { + void Clean() { if (_val != NULL) _val->subRef(); } public: - FastS_AutoRefCntPtr(void) : _val(NULL) { } + FastS_AutoRefCntPtr() : _val(NULL) { } explicit FastS_AutoRefCntPtr(T *val) {_val = val; } - ~FastS_AutoRefCntPtr(void) { Clean(); } + ~FastS_AutoRefCntPtr() { Clean(); } void Set(T *val) { Clean(); _val = val; } void SetDup(T *val) { Clean(); @@ -60,14 +60,14 @@ public: val->addRef(); _val = val; } - T *Get(void) const { return _val; } - T *GetDup(void) { + T *Get() const { return _val; } + T *GetDup() { if (_val != NULL) _val->addRef(); return _val; } - T *HandOver(void) { T *ret = _val; _val = NULL; return ret; } - void Drop(void) { + T *HandOver() { T *ret = _val; _val = NULL; return ret; } + void Drop() { if (_val != NULL) { _val->subRef(); _val = NULL; @@ -83,12 +83,12 @@ private: FastS_AutoCharPtr& operator=(const FastS_AutoCharPtr &); char *_val; - void Clean(void) { + void Clean() { if (_val != NULL) free(_val); } public: - FastS_AutoCharPtr(void) + FastS_AutoCharPtr() : _val(NULL) { } @@ -96,11 +96,11 @@ public: : _val(val) { } - ~FastS_AutoCharPtr(void) { Clean(); } + ~FastS_AutoCharPtr() { Clean(); } void Set(char *val) { Clean(); _val = val; } - char *Get(void) const { return _val; } - char *HandOver(void) { char *ret = _val; _val = NULL; return ret; } - void Drop(void) { + char *Get() const { return _val; } + char *HandOver() { char *ret = _val; _val = NULL; return ret; } + void Drop() { if (_val != NULL) { free(_val); _val = NULL; diff --git a/searchcore/src/vespa/searchcore/util/base64encoder.cpp b/searchcore/src/vespa/searchcore/util/base64encoder.cpp deleted file mode 100644 index 7b3938533a7..00000000000 --- a/searchcore/src/vespa/searchcore/util/base64encoder.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/searchcore/util/base64encoder.h> - -char FastS_Base64Encoder::_base64Padding = '='; - -char FastS_Base64Encoder::_base64Table[] = -{ - 'A','B','C','D','E','F','G','H','I','J','K','L','M', - 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z', - 'a','b','c','d','e','f','g','h','i','j','k','l','m', - 'n','o','p','q','r','s','t','u','v','w','x','y','z', - '0','1','2','3','4','5','6','7','8','9','+','/' -}; diff --git a/searchcore/src/vespa/searchcore/util/base64encoder.h b/searchcore/src/vespa/searchcore/util/base64encoder.h deleted file mode 100644 index 10fcd85abca..00000000000 --- a/searchcore/src/vespa/searchcore/util/base64encoder.h +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#pragma once - - -/** - * Simple class that may be used to base 64 encode a continuous data buffer. - **/ -class FastS_Base64Encoder -{ -private: - FastS_Base64Encoder(const FastS_Base64Encoder &); - FastS_Base64Encoder& operator=(const FastS_Base64Encoder &); - - /** - * The char used for padding in base 64 encoding. - **/ - static char _base64Padding; - - /** - * Table containing the 64 chars used to represent numbers from 0-63 - * in base 64 encoding. - **/ - static char _base64Table[]; - - - const unsigned char *_data; - const unsigned char *_dataPos; - const unsigned char *_dataEnd; - -public: - - - /** - * Create a base 64 encoder object with the task of encoding the - * given buffer. - * - * @param data the data to encode. - * @param datalen the byte-count of the data to encode. - **/ - FastS_Base64Encoder(const void *data, unsigned int datalen) - : _data(static_cast<const unsigned char *>(data)), - _dataPos(_data), - _dataEnd(_data + datalen) - { - } - - - /** - * @return number of bytes left in the input buffer. - **/ - unsigned int InputBytesLeft() - { - return (_dataEnd - _dataPos); - } - - - /** - * This method determines how much output space is needed to encode - * the rest of the input buffer referenced by this object. - * - * @return the space needed to encode the rest of the input. - **/ - unsigned int OutputBytesNeeded() - { - unsigned int groups = (_dataEnd - _dataPos) / 3; - if (((_dataEnd - _dataPos) % 3) != 0) - groups++; - - return (groups << 2); - } - - - /** - * Encode data from the buffer referenced by this object into the - * buffer given to this method. NOTE: dstLen should be at least 4 - * since this method only encodes in complete groups. - * - * @return the number of bytes of output generated. - * @param dst where to generate output. - * @param dstLen maximum output to generate. - **/ - unsigned int Encode(char *dst, unsigned int dstLen) - { - unsigned int groups = dstLen >> 2; - - char *dstPos = dst; - for (;groups > 0 && InputBytesLeft() >= 3; groups--) { - dstPos[0] = _base64Table[_dataPos[0] >> 2]; - dstPos[1] = _base64Table[((_dataPos[0] & 0x03) << 4) + (_dataPos[1] >> 4)]; - dstPos[2] = _base64Table[((_dataPos[1] & 0x0f) << 2) + (_dataPos[2] >> 6)]; - dstPos[3] = _base64Table[_dataPos[2] & 0x3f]; - dstPos += 4; - _dataPos += 3; - } - - if (groups > 0 && InputBytesLeft() > 0) { // handle padding - dstPos[0] = _base64Table[_dataPos[0] >> 2]; - - if (InputBytesLeft() == 2) { // 2 bytes left - dstPos[1] = _base64Table[((_dataPos[0] & 0x03) << 4) + (_dataPos[1] >> 4)]; - dstPos[2] = _base64Table[(_dataPos[1] & 0x0f) << 2]; - dstPos[3] = _base64Padding; - - } else { // 1 byte left - dstPos[1] = _base64Table[(_dataPos[0] & 0x03) << 4]; - dstPos[2] = _base64Padding; - dstPos[3] = _base64Padding; - } - dstPos += 4; - _dataPos = _dataEnd; - } - - return (dstPos - dst); - } - -}; - diff --git a/searchcore/src/vespa/searchcore/util/eventloop.cpp b/searchcore/src/vespa/searchcore/util/eventloop.cpp index fc249a2369a..a449dcf2007 100644 --- a/searchcore/src/vespa/searchcore/util/eventloop.cpp +++ b/searchcore/src/vespa/searchcore/util/eventloop.cpp @@ -1,14 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/searchcore/util/eventloop.h> +#include "eventloop.h" +#include <cstdio> double FastS_TimeOut::_val[FastS_TimeOut::valCnt]; - void FastS_TimeOut::WriteTime(char* buffer, size_t bufsize, double xtime) { diff --git a/searchcore/src/vespa/searchcore/util/eventloop.h b/searchcore/src/vespa/searchcore/util/eventloop.h index 1858352b07f..2b75f198448 100644 --- a/searchcore/src/vespa/searchcore/util/eventloop.h +++ b/searchcore/src/vespa/searchcore/util/eventloop.h @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once +#include <cstddef> class FastS_TimeOut { diff --git a/searchcore/src/vespa/searchcore/util/log.cpp b/searchcore/src/vespa/searchcore/util/log.cpp index 064d0ef3e92..2df60a50df3 100644 --- a/searchcore/src/vespa/searchcore/util/log.cpp +++ b/searchcore/src/vespa/searchcore/util/log.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> + #include <vespa/searchcore/util/log.h> + +#include <vespa/log/log.h> LOG_SETUP(".searchcore.util.log"); /** diff --git a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp b/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp index 29318809335..0b55ac58428 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp +++ b/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.flush.flushstats"); #include "flushstats.h" diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp index 73bda4d94c8..95c0df6cde6 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.activediskindexes"); - #include "activediskindexes.h" using std::set; using vespalib::string; using vespalib::LockGuard; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { void ActiveDiskIndexes::setActive(const string &index) { LockGuard lock(_lock); @@ -30,5 +25,4 @@ bool ActiveDiskIndexes::isActive(const string &index) const { return _active.find(index) != _active.end(); } -} // namespace index -} // namespace searchcorespi +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp index f9d5215d6f5..b627ddee271 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "disk_index_stats.h" #include "idiskindex.h" -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { DiskIndexStats::DiskIndexStats() : IndexSearchableStats(), @@ -24,5 +22,4 @@ DiskIndexStats::~DiskIndexStats() { } -} // namespace searchcorespi::index -} // namespace searchcorespi +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index 68e7853c820..d2788ed558a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -1,21 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.diskindexcleaner"); - -#include "activediskindexes.h" #include "diskindexcleaner.h" -#include <algorithm> +#include "activediskindexes.h" +#include <vespa/fastos/file.h> #include <sstream> #include <vector> +#include <vespa/log/log.h> +LOG_SETUP(".searchcorespi.index.diskindexcleaner"); + using std::istringstream; using vespalib::string; using std::vector; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { + namespace { vector<string> readIndexes(const string &base_dir) { vector<string> indexes; @@ -109,5 +108,5 @@ void DiskIndexCleaner::removeOldIndexes( vector<string> indexes = readIndexes(base_dir); removeOld(base_dir, indexes, active_indexes); } -} // namespace index -} // namespace searchcorespi + +}
\ No newline at end of file diff --git a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp index c5e4a382233..4a640f16389 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.eventlogger"); #include "eventlogger.h" #include <vespa/searchlib/util/logutil.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchcorespi.index.eventlogger"); + using vespalib::JSONStringer; using search::util::LogUtil; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { void EventLogger::diskIndexLoadStart(const vespalib::string &indexDir) @@ -67,5 +66,4 @@ EventLogger::diskFusionComplete(const vespalib::string &fusionDir, EV_STATE("fusion.complete", jstr.toString().c_str()); } -} // namespace index -} // namespace searchcorespi +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp b/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp index 84c34714526..8c7a1851f42 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.fusionrunner"); - #include "fusionrunner.h" #include "eventlogger.h" #include "fusionspec.h" @@ -13,6 +9,9 @@ LOG_SETUP(".searchcorespi.index.fusionrunner"); #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/util/jsonwriter.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchcorespi.index.fusionrunner"); + using search::FixedSourceSelector; using search::TuneFileAttributes; using search::TuneFileIndexing; @@ -26,8 +25,7 @@ using std::vector; using vespalib::string; using vespalib::JSONStringer; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { FusionRunner::FusionRunner(const string &base_dir, const Schema &schema, @@ -130,5 +128,4 @@ FusionRunner::fuse(const FusionSpec &fusion_spec, return fusion_id; } -} // namespace index -} // namespace searchcorespi +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h b/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h index 5e31698a7fc..f5c6006eb2d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h +++ b/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h @@ -2,10 +2,10 @@ #pragma once +#include <cstdint> #include <vector> -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { /** * Specifies a set of disk index ids for fusion. @@ -19,6 +19,4 @@ struct FusionSpec { FusionSpec() : last_fusion_id(0), flush_ids() {} }; -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp index 7920c1650cb..ed2e4601963 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.index_manager_explorer"); #include "index_manager_explorer.h" #include "index_manager_stats.h" diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp index 035a5698c2f..6493fb7fcaa 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "index_manager_stats.h" #include "iindexmanager.h" #include "indexsearchablevisitor.h" - namespace searchcorespi { namespace { diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h index 048124b688f..637dcafdfbe 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h @@ -3,6 +3,7 @@ #include "disk_index_stats.h" #include "memory_index_stats.h" +#include <vector> namespace searchcorespi { diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp index c5ce5163aac..1d72340e79c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "index_searchable_stats.h" #include "indexsearchable.h" -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { IndexSearchableStats::IndexSearchableStats() : _serialNum(0), @@ -26,4 +24,3 @@ bool IndexSearchableStats::operator<(const IndexSearchableStats &rhs) const } } // namespace searchcorespi::index -} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp index f886f7eb536..a0fdefe7e8b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp @@ -1,13 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.indexdisklayout"); #include "indexdisklayout.h" #include <sstream> -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { const vespalib::string IndexDiskLayout::FlushDirPrefix = vespalib::string("index.flush."); @@ -57,6 +53,4 @@ IndexDiskLayout::getSelectorFileName(const vespalib::string &dir) return dir + "/selector"; } -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp index 84ce5b77bef..1384d5bf868 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.indexmaintainerconfig"); #include "indexmaintainerconfig.h" using search::index::Schema; using search::TuneFileAttributes; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { IndexMaintainerConfig::IndexMaintainerConfig(const vespalib::string &baseDir, const WarmupConfig & warmup, @@ -26,6 +22,6 @@ IndexMaintainerConfig::IndexMaintainerConfig(const vespalib::string &baseDir, { } -} // namespace index -} // namespace searchcorespi +IndexMaintainerConfig::~IndexMaintainerConfig() { } +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h index ea51399bcd7..58c58af7328 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcommon/common/schema.h> +#include "warmupconfig.h" #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/stllike/string.h> -#include "warmupconfig.h" -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { /** * Class that keeps the config used when constructing an index maintainer. @@ -30,6 +29,8 @@ public: const search::SerialNum serialNum, const search::TuneFileAttributes &tuneFileAttributes); + ~IndexMaintainerConfig(); + /** * Returns the base directory in which the maintainer will store its indexes. */ @@ -60,7 +61,4 @@ public: } }; -} // namespace index -} // namespace searchcorespi - - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp index d18920f5c7a..c04815c8bec 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.indexmaintainercontext"); #include "indexmaintainercontext.h" @@ -9,8 +6,7 @@ using search::common::FileHeaderContext; using search::TuneFileAttributes; using searchcorespi::IIndexManager; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { IndexMaintainerContext::IndexMaintainerContext(IThreadingService &threadingService, IIndexManager::Reconfigurer &reconfigurer, @@ -23,6 +19,4 @@ IndexMaintainerContext::IndexMaintainerContext(IThreadingService &threadingServi { } -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h index ee8087bef99..4804e0a77f5 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h @@ -2,13 +2,12 @@ #pragma once #include "ithreadingservice.h" -#include <vespa/searchcorespi/index/iindexmanager.h> +#include "iindexmanager.h" #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/util/threadexecutor.h> -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { /** * Class that keeps the long-lived context used by an index maintainer. @@ -53,7 +52,4 @@ public: vespalib::ThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; } }; -} // namespace index -} // namespace searchcorespi - - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp index 830e4f68c45..8d6bf807049 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.indexmanagerconfig"); #include "indexmanagerconfig.h" @@ -16,5 +13,7 @@ IndexManagerConfig::IndexManagerConfig(const vespalib::string &configId, { } +IndexManagerConfig::~IndexManagerConfig() { } + } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h b/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h index 7019a10668a..1a6ff8f1cba 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h @@ -19,6 +19,7 @@ public: IndexManagerConfig(const vespalib::string &configId, const config::ConfigSnapshot &configSnapshot, size_t numSearcherThreads); + ~IndexManagerConfig(); /** * Returns the config id used to retrieve the configs from the config snapshot instance. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp index d2c76612833..0f002c7282f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchcorespi.index.indexreadutilities"); #include "indexreadutilities.h" #include "indexdisklayout.h" @@ -10,11 +7,13 @@ LOG_SETUP(".searchcorespi.index.indexreadutilities"); #include <set> #include <vector> +#include <vespa/log/log.h> +LOG_SETUP(".searchcorespi.index.indexreadutilities"); + using search::SerialNum; using vespalib::FileHeader; -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { namespace { @@ -86,6 +85,4 @@ IndexReadUtilities::readSerialNum(const vespalib::string &dir) return 0; } -} // namespace index -} // namespace searchcorespi - +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp index 7805702fd03..2b090e7fd62 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "memory_index_stats.h" #include "imemoryindex.h" - -namespace searchcorespi { -namespace index { +namespace searchcorespi::index { MemoryIndexStats::MemoryIndexStats() : IndexSearchableStats() @@ -23,4 +20,3 @@ MemoryIndexStats::~MemoryIndexStats() } } // namespace searchcorespi::index -} // namespace searchcorespi diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 58d32020fb5..b9e0a8f8943 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -58,11 +58,10 @@ vespa_define_module( APPS src/apps/docstore - src/apps/expgolomb - src/apps/fileheaderinspect - src/apps/loadattribute src/apps/tests src/apps/uniform + src/apps/vespa-attribute-inspect + src/apps/vespa-fileheader-inspect src/apps/vespa-index-inspect src/apps/vespa-ranking-expression-analyzer @@ -117,6 +116,7 @@ vespa_define_module( src/tests/docstore/chunk src/tests/docstore/document_store src/tests/docstore/document_store_visitor + src/tests/docstore/file_chunk src/tests/docstore/lid_info src/tests/docstore/logdatastore src/tests/docstore/store_by_bucket @@ -155,7 +155,6 @@ vespa_define_module( src/tests/fef/table src/tests/fef/termfieldmodel src/tests/fef/termmatchdatamerger - src/tests/fileheaderinspect src/tests/fileheadertk src/tests/forcelink src/tests/grouping @@ -220,4 +219,5 @@ vespa_define_module( src/tests/util/slime_output_raw_buf_adapter src/tests/util/statebuf src/tests/util/statefile + src/tests/vespa-fileheader-inspect ) diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 2393a82b44c..87cfc219d2e 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -16,23 +16,23 @@ using namespace search; class BenchmarkDataStoreApp : public FastOS_Application { - void usage(void); + void usage(); int benchmark(const vespalib::string & directory, size_t numReads, size_t numThreads, size_t perChunk, const vespalib::string & readType); - int Main(void) override; + int Main() override; void read(size_t numReads, size_t perChunk, const IDataStore * dataStore); }; void -BenchmarkDataStoreApp::usage(void) +BenchmarkDataStoreApp::usage() { printf("Usage: %s <direcory> <numreads> <numthreads> <objects per read> <normal,directio,mmap,mlock>\n", _argv[0]); fflush(stdout); } int -BenchmarkDataStoreApp::Main(void) +BenchmarkDataStoreApp::Main() { if (_argc >= 2) { size_t numThreads(16); @@ -68,15 +68,15 @@ void BenchmarkDataStoreApp::read(size_t numReads, size_t perChunk, const IDataSt char state[8]; memset(state, 0, sizeof(state)); memset(&rstate, 0, sizeof(rstate)); - const size_t numDocs(dataStore->nextId()); - assert(numDocs > 0); + const size_t docIdLimit(dataStore->getDocIdLimit()); + assert(docIdLimit > 0); initstate_r(getpid(), state, sizeof(state), &rstate); assert(srandom_r(getpid(), &rstate) == 0); int32_t rnd(0); for ( size_t i(0); i < numReads; i++) { random_r(&rstate, &rnd); - uint32_t lid(rnd%numDocs); - for (uint32_t j(lid); j < std::min(numDocs, lid+perChunk); j++) { + uint32_t lid(rnd%docIdLimit); + for (uint32_t j(lid); j < std::min(docIdLimit, lid+perChunk); j++) { dataStore->read(j, buf); buf.clear(); } diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 2e85bb168cb..fdb05dabf62 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -11,13 +11,13 @@ using namespace search; class CreateIdxFileFromDatApp : public FastOS_Application { - void usage(void); + void usage(); int createIdxFile(const vespalib::string & datFileName, const vespalib::string & idxFileName); - int Main(void) override; + int Main() override; }; void -CreateIdxFileFromDatApp::usage(void) +CreateIdxFileFromDatApp::usage() { printf("Usage: %s <datfile> <idxfile>\n", _argv[0]); fflush(stdout); @@ -95,7 +95,7 @@ int CreateIdxFileFromDatApp::createIdxFile(const vespalib::string & datFileName, FastOS_File idxFile(idxFileName.c_str()); assert(idxFile.OpenWriteOnly()); index::DummyFileHeaderContext fileHeaderContext; - idxFile.SetPosition(WriteableFileChunk::writeIdxHeader(fileHeaderContext, idxFile)); + idxFile.SetPosition(WriteableFileChunk::writeIdxHeader(fileHeaderContext, std::numeric_limits<uint32_t>::max(), idxFile)); fprintf(stdout, "datHeaderLen=%ld\n", datHeaderLen); uint64_t serialNum(0); for (const char * current(start + datHeaderLen); current < end; ) { @@ -149,7 +149,7 @@ int CreateIdxFileFromDatApp::createIdxFile(const vespalib::string & datFileName, } int -CreateIdxFileFromDatApp::Main(void) +CreateIdxFileFromDatApp::Main() { vespalib::string cmd; if (_argc == 3) { diff --git a/searchlib/src/apps/docstore/documentstoreinspect.cpp b/searchlib/src/apps/docstore/documentstoreinspect.cpp index ffd23f30dba..42ff9e1395f 100644 --- a/searchlib/src/apps/docstore/documentstoreinspect.cpp +++ b/searchlib/src/apps/docstore/documentstoreinspect.cpp @@ -11,16 +11,16 @@ using namespace search; class DocumentStoreInspectApp : public FastOS_Application { - void usage(void); + void usage(); int verify(const vespalib::string & directory); int dumpIdxFile(const vespalib::string & file); - int Main(void) override; + int Main() override; }; void -DocumentStoreInspectApp::usage(void) +DocumentStoreInspectApp::usage() { printf("Usage: %s dumpidxfile [--idxfile idxFile]\n", _argv[0]); fflush(stdout); @@ -33,7 +33,8 @@ int DocumentStoreInspectApp::dumpIdxFile(const vespalib::string & file) if (idxFile.OpenReadOnly()) { if (idxFile.IsMemoryMapped()) { int64_t fileSize = idxFile.GetSize(); - uint64_t idxHeaderLen = FileChunk::readIdxHeader(idxFile); + uint32_t docIdLimit = std::numeric_limits<uint32_t>::max(); + uint64_t idxHeaderLen = FileChunk::readIdxHeader(idxFile, docIdLimit); vespalib::nbostream is(static_cast<const char *> (idxFile.MemoryMapPtr(0)) + idxHeaderLen, fileSize - idxHeaderLen); @@ -61,7 +62,7 @@ int DocumentStoreInspectApp::dumpIdxFile(const vespalib::string & file) } int -DocumentStoreInspectApp::Main(void) +DocumentStoreInspectApp::Main() { vespalib::string cmd; if (_argc >= 2) { diff --git a/searchlib/src/apps/docstore/verifylogdatastore.cpp b/searchlib/src/apps/docstore/verifylogdatastore.cpp index e586b1b4ed7..a3d2034604d 100644 --- a/searchlib/src/apps/docstore/verifylogdatastore.cpp +++ b/searchlib/src/apps/docstore/verifylogdatastore.cpp @@ -12,22 +12,22 @@ using namespace search; class VerifyLogDataStoreApp : public FastOS_Application { - void usage(void); + void usage(); int verify(const vespalib::string & directory); - int Main(void) override; + int Main() override; }; void -VerifyLogDataStoreApp::usage(void) +VerifyLogDataStoreApp::usage() { printf("Usage: %s <direcory>\n", _argv[0]); fflush(stdout); } int -VerifyLogDataStoreApp::Main(void) +VerifyLogDataStoreApp::Main() { if (_argc >= 2) { vespalib::string directory(_argv[1]); diff --git a/searchlib/src/apps/expgolomb/.gitignore b/searchlib/src/apps/expgolomb/.gitignore deleted file mode 100644 index 0886ab154a2..00000000000 --- a/searchlib/src/apps/expgolomb/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -expgolomb diff --git a/searchlib/src/apps/expgolomb/expgolomb.cpp b/searchlib/src/apps/expgolomb/expgolomb.cpp deleted file mode 100644 index 4d783d8eaa4..00000000000 --- a/searchlib/src/apps/expgolomb/expgolomb.cpp +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vector> - -class ExpGolombApp : public FastOS_Application -{ - void - usage(void); - - int - testExpGolomb64(int kValue); - - int - testExpGolomb64le(int kValue); - - int - Main(void) override; -}; - - - -void -ExpGolombApp::usage(void) -{ - printf("Usage: expgolomb testeg64 <kValue>]\n"); - fflush(stdout); -} - - -int -ExpGolombApp::testExpGolomb64(int kValue) -{ - std::vector<uint64_t> myrand; - for (int i = 0; i < 10000; ++i) { - uint64_t rval = rand(); - rval <<= 30; - rval |= rand(); - myrand.push_back(rval); - } - for (int i = 0; i < 10000; ++i) { - uint64_t rval = rand(); - rval <<= 30; - rval |= rand(); - uint32_t bits = (rand() & 63); - rval &= ((UINT64_C(1) << bits) - 1); - myrand.push_back(rval); - } - typedef search::bitcompression::EncodeContext64BE EC; - - EC e; - search::ComprFileWriteContext wc(e); - wc.allocComprBuf(32768, 32768); - e.setupWrite(wc); - - int rsize = myrand.size(); - for (int i = 0; i < rsize; ++i) { - e.encodeExpGolomb(myrand[i], kValue); - if (e._valI >= e._valE) - wc.writeComprBuffer(false); - } - e.flush(); - - UC64_DECODECONTEXT(o); - unsigned int length; - uint64_t val64; - UC64BE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC); - - bool failure = false; - for (int i = 0; i < rsize; ++i) { - UC64BE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt, - kValue, EC); - if (val64 != myrand[i]) { - printf("FAILURE: TestExpGolomb64, val64=%" - PRIu64 ", myrand[%d]=%" PRIu64 "\n", - val64, i, myrand[i]); - failure = true; - } - } - if (!failure) - printf("SUCCESS: TestExpGolomb64\n"); - return failure ? 1 : 0; -} - -int -ExpGolombApp::testExpGolomb64le(int kValue) -{ - std::vector<uint64_t> myrand; - for (int i = 0; i < 10000; ++i) { - uint64_t rval = rand(); - rval <<= 30; - rval |= rand(); - myrand.push_back(rval); - } - for (int i = 0; i < 10000; ++i) { - uint64_t rval = rand(); - rval <<= 30; - rval |= rand(); - uint32_t bits = (rand() & 63); - rval &= ((UINT64_C(1) << bits) - 1); - myrand.push_back(rval); - } - typedef search::bitcompression::EncodeContext64LE EC; - - EC e; - search::ComprFileWriteContext wc(e); - wc.allocComprBuf(32768, 32768); - e.setupWrite(wc); - - int rsize = myrand.size(); - for (int i = 0; i < rsize; ++i) { - e.encodeExpGolomb(myrand[i], kValue); - if (e._valI >= e._valE) - wc.writeComprBuffer(false); - } - e.flush(); - - UC64_DECODECONTEXT(o); - unsigned int length; - uint64_t val64; - UC64LE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC); - - bool failure = false; - for (int i = 0; i < rsize; ++i) { - UC64LE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt, - kValue, EC); - if (val64 != myrand[i]) { - printf("FAILURE: TestExpGolomb64le, val64=%" - PRIu64 ", myrand[%d]=%" PRIu64 "\n", - val64, i, myrand[i]); - failure = true; - } - } - if (!failure) - printf("SUCCESS: TestExpGolomb64le\n"); - return failure ? 1 : 0; -} - - -int -ExpGolombApp::Main(void) -{ - printf("Hello world\n"); - if (_argc >= 2) { - if (strcmp(_argv[1], "testeg64") == 0) { - if (_argc < 3) { - fprintf(stderr, "Too few arguments\n"); - usage(); - return 1; - } - return testExpGolomb64(atoi(_argv[2])); - } else if (strcmp(_argv[1], "testeg64le") == 0) { - if (_argc < 3) { - fprintf(stderr, "Too few arguments\n"); - usage(); - return 1; - } - return testExpGolomb64le(atoi(_argv[2])); - } else { - fprintf(stderr, "Wrong arguments\n"); - usage(); - return 1; - } - } else { - fprintf(stderr, "Too few arguments\n"); - usage(); - return 1; - } - return 0; -} - -FASTOS_MAIN(ExpGolombApp); - - diff --git a/searchlib/src/apps/fileheaderinspect/.gitignore b/searchlib/src/apps/fileheaderinspect/.gitignore deleted file mode 100644 index 5616f8e735c..00000000000 --- a/searchlib/src/apps/fileheaderinspect/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -vespa-header-inspect diff --git a/searchlib/src/apps/fileheaderinspect/CMakeLists.txt b/searchlib/src/apps/fileheaderinspect/CMakeLists.txt deleted file mode 100644 index 322bf6fefcb..00000000000 --- a/searchlib/src/apps/fileheaderinspect/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_vespa-header-inspect_app - SOURCES - fileheaderinspect.cpp - OUTPUT_NAME vespa-header-inspect - INSTALL bin - DEPENDS - searchlib -) diff --git a/searchlib/src/apps/loadattribute/.gitignore b/searchlib/src/apps/loadattribute/.gitignore deleted file mode 100644 index 4f008fbf84e..00000000000 --- a/searchlib/src/apps/loadattribute/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -loadattribute diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index 7e4cdf32fdd..4db392e31db 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("btreestress_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 60e70566c9e..d69bc09d010 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/memoryindex/memoryindex.h> #include <vespa/searchlib/fef/matchdata.h> @@ -70,7 +69,7 @@ makeSchema() } document::DocumenttypesConfig -makeDocTypeRepoConfig(void) +makeDocTypeRepoConfig() { const int32_t doc_type_id = 787121340; document::config_builder::DocumenttypesConfigBuilderHelper builder; diff --git a/searchlib/src/apps/uniform/.gitignore b/searchlib/src/apps/uniform/.gitignore index ff18dbaa7fd..d1fbbf62d24 100644 --- a/searchlib/src/apps/uniform/.gitignore +++ b/searchlib/src/apps/uniform/.gitignore @@ -1,3 +1,3 @@ .depend Makefile -uniform +searchlib_uniform_app diff --git a/searchlib/src/apps/uniform/CMakeLists.txt b/searchlib/src/apps/uniform/CMakeLists.txt index 9f9c2139f42..7b835a64e8c 100644 --- a/searchlib/src/apps/uniform/CMakeLists.txt +++ b/searchlib/src/apps/uniform/CMakeLists.txt @@ -2,8 +2,6 @@ vespa_add_executable(searchlib_uniform_app SOURCES uniform.cpp - OUTPUT_NAME uniform - INSTALL bin DEPENDS searchlib ) diff --git a/searchlib/src/apps/uniform/uniform.cpp b/searchlib/src/apps/uniform/uniform.cpp index 356c5e832aa..8fb6b2af0ca 100644 --- a/searchlib/src/apps/uniform/uniform.cpp +++ b/searchlib/src/apps/uniform/uniform.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include <vespa/fastos/app.h> #include <vespa/searchlib/bitcompression/compression.h> @@ -23,25 +23,15 @@ class UniformApp : public FastOS_Application uint64_t _bits[MAXK + 1]; uint64_t _next; - static uint32_t - encodeSpace(uint64_t x, uint32_t k) - { - return EC64::encodeExpGolombSpace(x, k); - } - - void - clearBits(void); - - void - reportBits(void); - - int - Main(void) override; + static uint32_t encodeSpace(uint64_t x, uint32_t k) { return EC64::encodeExpGolombSpace(x, k); } + void clearBits(); + void reportBits(); + int Main() override; }; void -UniformApp::clearBits(void) +UniformApp::clearBits() { for (unsigned int k = 0; k <= MAXK; ++k) _bits[k] = 0; @@ -50,7 +40,7 @@ UniformApp::clearBits(void) void -UniformApp::reportBits(void) +UniformApp::reportBits() { printf("next=%" PRIu64 " ", _next); for (unsigned int k = 0; k <= MAXK; ++k) @@ -64,7 +54,7 @@ UniformApp::reportBits(void) int -UniformApp::Main(void) +UniformApp::Main() { int k, l, m, bestmask, oldbestmask; printf("Hello world\n"); diff --git a/searchlib/src/apps/vespa-attribute-inspect/.gitignore b/searchlib/src/apps/vespa-attribute-inspect/.gitignore new file mode 100644 index 00000000000..62bcd8ab443 --- /dev/null +++ b/searchlib/src/apps/vespa-attribute-inspect/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +vespa-attribute-inspect diff --git a/searchlib/src/apps/loadattribute/CMakeLists.txt b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt index 6712519e59a..4d170b9f325 100644 --- a/searchlib/src/apps/loadattribute/CMakeLists.txt +++ b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt @@ -1,8 +1,8 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_loadattribute_app +vespa_add_executable(searchlib_vespa-attribute-inspect_app SOURCES - loadattribute.cpp - OUTPUT_NAME loadattribute + vespa-attribute-inspect.cpp + OUTPUT_NAME vespa-attribute-inspect INSTALL bin DEPENDS searchlib diff --git a/searchlib/src/apps/loadattribute/loadattribute.rb b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb index d1fb5a5632c..d1fb5a5632c 100644 --- a/searchlib/src/apps/loadattribute/loadattribute.rb +++ b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb diff --git a/searchlib/src/apps/loadattribute/loadattribute.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp index e82ab3ac645..189074aa5d3 100644 --- a/searchlib/src/apps/loadattribute/loadattribute.cpp +++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp @@ -98,7 +98,7 @@ LoadAttribute::printContent(const AttributePtr & ptr, std::ostream & os) void LoadAttribute::usage() { - std::cout << "usage: loadattribute [-p (print content to <attribute>.out)]" << std::endl; + std::cout << "usage: vespa-attribute-inspect [-p (print content to <attribute>.out)]" << std::endl; std::cout << " [-a (apply a single update)]" << std::endl; std::cout << " [-s (save attribute to <attribute>.save.dat)]" << std::endl; std::cout << " <attribute>" << std::endl; diff --git a/searchlib/src/apps/vespa-fileheader-inspect/.gitignore b/searchlib/src/apps/vespa-fileheader-inspect/.gitignore new file mode 100644 index 00000000000..5b53dd5e3ac --- /dev/null +++ b/searchlib/src/apps/vespa-fileheader-inspect/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +vespa-fileheader-inspect diff --git a/searchlib/src/apps/expgolomb/CMakeLists.txt b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt index 230718907dd..cbbe8c05c2b 100644 --- a/searchlib/src/apps/expgolomb/CMakeLists.txt +++ b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt @@ -1,8 +1,8 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_expgolomb_app +vespa_add_executable(searchlib_vespa-fileheader-inspect_app SOURCES - expgolomb.cpp - OUTPUT_NAME expgolomb + vespa-fileheader-inspect.cpp + OUTPUT_NAME vespa-fileheader-inspect INSTALL bin DEPENDS searchlib diff --git a/searchlib/src/apps/fileheaderinspect/fileheaderinspect.cpp b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp index 2407be0b4c2..02cbb230960 100644 --- a/searchlib/src/apps/fileheaderinspect/fileheaderinspect.cpp +++ b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp @@ -7,7 +7,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/log.h> -LOG_SETUP("fileheaderinspect"); +LOG_SETUP("vespa-fileheader-inspect"); using namespace vespalib; diff --git a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp index 99049c0dea7..d0f7c8e1623 100644 --- a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp +++ b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/bitcompression/compression.h> @@ -15,7 +14,9 @@ #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/fastos/app.h> #include <iostream> + #include <vespa/log/log.h> LOG_SETUP("vespa-index-inspect"); @@ -41,8 +42,7 @@ using search::index::schema::DataType; using search::queryeval::SearchIterator; using namespace search::index; -namespace -{ +namespace { /** * Fine granularity, for small scale inversion within a single document. @@ -118,7 +118,7 @@ unpackFeatures(std::vector<PosEntry> &entries, void -usageHeader(void) +usageHeader() { using std::cerr; cerr << @@ -140,20 +140,9 @@ public: { } - void - addField(const vespalib::string &field) - { - _fields.push_back(field); - } - - bool - empty(void) const - { - return _ids.empty(); - } - - void - validateFields(const Schema &schema); + void addField(const vespalib::string &field) { _fields.push_back(field); } + bool empty() const { return _ids.empty(); } + void validateFields(const Schema &schema); }; @@ -188,19 +177,10 @@ public: { } - virtual - ~SubApp(void) - { - } - - virtual void - usage(bool showHeader) = 0; - - virtual bool - getOptions(void) = 0; - - virtual int - run(void) = 0; + virtual ~SubApp() { } + virtual void usage(bool showHeader) = 0; + virtual bool getOptions() = 0; + virtual int run() = 0; }; @@ -220,51 +200,20 @@ class ShowPostingListSubApp : public SubApp uint32_t _docIdLimit; uint32_t _minDocId; - static uint64_t - noWordNumHigh(void) - { - return std::numeric_limits<uint64_t>::max(); - } - - static uint64_t - noWordNum(void) - { - return 0u; - } + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } + static uint64_t noWordNum() { return 0u; } public: - ShowPostingListSubApp(FastOS_Application &app); - - virtual - ~ShowPostingListSubApp(void); - - virtual void - usage(bool showHeader) override; - - virtual bool - getOptions(void) override; - - virtual int - run(void) override; - - void - showPostingList(void); - - bool - readDocIdLimit(const Schema &schema); - - bool - readWordList(const SchemaUtil::IndexIterator &index); - - bool - readWordList(const Schema &schema); - - void - readPostings(const SchemaUtil::IndexIterator &index, - std::vector<PosEntry> &entries); - - void - showTransposedPostingList(); + virtual ~ShowPostingListSubApp(); + virtual void usage(bool showHeader) override; + virtual bool getOptions() override; + virtual int run() override; + void showPostingList(); + bool readDocIdLimit(const Schema &schema); + bool readWordList(const SchemaUtil::IndexIterator &index); + bool readWordList(const Schema &schema); + void readPostings(const SchemaUtil::IndexIterator &index, std::vector<PosEntry> &entries); + void showTransposedPostingList(); }; @@ -287,7 +236,7 @@ ShowPostingListSubApp::ShowPostingListSubApp(FastOS_Application &app) } -ShowPostingListSubApp::~ShowPostingListSubApp(void) +ShowPostingListSubApp::~ShowPostingListSubApp() { } @@ -312,7 +261,7 @@ ShowPostingListSubApp::usage(bool showHeader) bool -ShowPostingListSubApp::getOptions(void) +ShowPostingListSubApp::getOptions() { int c; const char *optArgument = NULL; @@ -509,7 +458,7 @@ ShowPostingListSubApp::readPostings(const SchemaUtil::IndexIterator &index, void -ShowPostingListSubApp::showTransposedPostingList(void) +ShowPostingListSubApp::showTransposedPostingList() { Schema schema; std::string schemaName = _indexDir + "/schema.txt"; @@ -579,7 +528,7 @@ ShowPostingListSubApp::showTransposedPostingList(void) void -ShowPostingListSubApp::showPostingList(void) +ShowPostingListSubApp::showPostingList() { Schema schema; uint32_t numFields = 1; @@ -715,7 +664,7 @@ ShowPostingListSubApp::showPostingList(void) int -ShowPostingListSubApp::run(void) +ShowPostingListSubApp::run() { if (_transpose) showTransposedPostingList(); @@ -736,21 +685,11 @@ class DumpWordsSubApp : public SubApp public: DumpWordsSubApp(FastOS_Application &app); - - virtual - ~DumpWordsSubApp(void); - - virtual void - usage(bool showHeader) override; - - virtual bool - getOptions(void) override; - - virtual int - run(void) override; - - void - dumpWords(void); + virtual ~DumpWordsSubApp(); + virtual void usage(bool showHeader) override; + virtual bool getOptions() override; + virtual int run() override; + void dumpWords(); }; @@ -765,7 +704,7 @@ DumpWordsSubApp::DumpWordsSubApp(FastOS_Application &app) } -DumpWordsSubApp::~DumpWordsSubApp(void) +DumpWordsSubApp::~DumpWordsSubApp() { } @@ -785,7 +724,7 @@ DumpWordsSubApp::usage(bool showHeader) bool -DumpWordsSubApp::getOptions(void) +DumpWordsSubApp::getOptions() { int c; const char *optArgument = NULL; @@ -853,7 +792,7 @@ DumpWordsSubApp::getOptions(void) void -DumpWordsSubApp::dumpWords(void) +DumpWordsSubApp::dumpWords() { search::index::Schema schema; std::string schemaName = _indexDir + "/schema.txt"; @@ -907,7 +846,7 @@ DumpWordsSubApp::dumpWords(void) int -DumpWordsSubApp::run(void) +DumpWordsSubApp::run() { dumpWords(); return 0; @@ -917,24 +856,20 @@ DumpWordsSubApp::run(void) class VespaIndexInspectApp : public FastOS_Application { public: - VespaIndexInspectApp(void); - - void - usage(void); - - int - Main(void) override; + VespaIndexInspectApp(); + void usage(); + int Main() override; }; -VespaIndexInspectApp::VespaIndexInspectApp(void) +VespaIndexInspectApp::VespaIndexInspectApp() : FastOS_Application() { } void -VespaIndexInspectApp::usage(void) +VespaIndexInspectApp::usage() { ShowPostingListSubApp(*this).usage(true); DumpWordsSubApp(*this).usage(false); @@ -942,7 +877,7 @@ VespaIndexInspectApp::usage(void) int -VespaIndexInspectApp::Main(void) +VespaIndexInspectApp::Main() { if (_argc < 2) { usage(); diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp index e97887c107e..ccfbf4c0e14 100644 --- a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp +++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp @@ -1,23 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <map> #include <vespa/searchlib/features/rankingexpression/feature_name_extractor.h> -#include <vector> #include <vespa/eval/eval/llvm/compiled_function.h> -#include <vespa/eval/eval/function.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/basic_nodes.h> #include <vespa/eval/eval/call_nodes.h> #include <vespa/eval/eval/operator_nodes.h> #include <vespa/vespalib/util/benchmark_timer.h> -#include <vespa/eval/eval/gbdt.h> #include <vespa/eval/eval/vm_forest.h> #include <vespa/eval/eval/llvm/deinline_forest.h> #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/eval/eval/param_usage.h> -#include <cmath> +#include <vespa/fastos/app.h> //----------------------------------------------------------------------------- diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java index 8c5db8a6ecc..ded765d5770 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.objects.Serializer; /** * This class represents a generic hit with a rank value. Actual hits are represented using subclasses of this class. * - * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author havardpe */ public abstract class Hit extends Identifiable { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java index c9e29c86f66..f2e994077a9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java @@ -14,9 +14,9 @@ import java.util.List; /** * This is an aggregated result holding the top n hits for a single group. * - * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author havardpe * @author baldersheim - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class HitsAggregationResult extends AggregationResult { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/QueryTerm.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/QueryTerm.java index 47ffb276f70..be50b413eba 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/QueryTerm.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/QueryTerm.java @@ -51,15 +51,18 @@ public final class QueryTerm { /** The degree to which this is exactly the term the user specified (1), or a stemmed form (closer to 0) */ public float getExactness() { return exactness; } - public @Override int hashCode() { return term.hashCode(); } + @Override + public int hashCode() { return term.hashCode(); } - public @Override boolean equals(Object object) { + @Override + public boolean equals(Object object) { if (! (object instanceof QueryTerm)) return false; return this.term.equals(((QueryTerm)object).term); } - public @Override String toString() { + @Override + public String toString() { if (connectedness==0.1f) return term; return connectedness + ":" + term; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java index 2dffe2a1100..2862a4783a2 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java @@ -44,7 +44,8 @@ public class MapContext extends Context { /** * Returns the value of a key. 0 is returned if the given key is not bound in this. */ - public @Override Value get(String key) { + @Override + public Value get(String key) { Value value=bindings.get(key); if (value==null) return DoubleValue.zero; return value; @@ -56,7 +57,8 @@ public class MapContext extends Context { * * @since 5.1.5 */ - public @Override void put(String key,Value value) { + @Override + public void put(String key,Value value) { bindings.put(key,value.freeze()); } @@ -70,12 +72,14 @@ public class MapContext extends Context { public MapContext thawedCopy() { return new MapContext(new HashMap<>(bindings)); } /** Returns an unmodifiable map of the names of this */ - public @Override Set<String> names() { + @Override + public Set<String> names() { if (frozen) return bindings.keySet(); return Collections.unmodifiableMap(bindings).keySet(); } - public @Override String toString() { + @Override + public String toString() { return "a map context [" + bindings.size() + " bindings]"; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index 6d038113691..082927456f4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -31,7 +31,7 @@ public class TensorValue extends Value { @Override public double asDouble() { if (hasDouble()) - return value.get(TensorAddress.empty); + return value.get(TensorAddress.of()); throw new UnsupportedOperationException("Requires a double value, but " + this.value + " cannot be " + "used as a double"); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/.gitignore b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/.gitignore +++ /dev/null diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/test/.gitignore b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/test/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/test/.gitignore +++ /dev/null diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/mlr/.gitignore b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/mlr/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/mlr/.gitignore +++ /dev/null diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java index 1e3b0c4362d..4e40ff7437e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java @@ -44,7 +44,7 @@ public class GeneratorLambdaFunctionNode extends CompositeNode { @Override public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) { - return (type + "(" + generator.toString(context, path, this)) + ")"; + return generator.toString(context, path, this); } /** Evaluate this in a context which must have the arguments bound */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java index 932975f3b63..234825eaa3d 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java @@ -27,7 +27,8 @@ public enum TruthOperator implements Serializable { /** Perform the truth operation on the input */ public abstract boolean evaluate(double x, double y); - public @Override String toString() { return operatorString; } + @Override + public String toString() { return operatorString; } public static TruthOperator fromString(String string) { for (TruthOperator operator : values()) diff --git a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java index 14ea58961ba..2663fc26b1a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java @@ -22,7 +22,8 @@ public class SemanticDistanceTestCase extends junit.framework.TestCase { super(name); } - public @Override void setUp() { + @Override + public void setUp() { c=new FieldMatchMetricsComputer(); StringBuilder field=new StringBuilder(); for (int i=0; i<150; i++) diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index d0a188c0760..deaf1070d45 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -265,6 +265,16 @@ public class EvaluationTestCase { tester.assertEvaluates("1.0", "sum(tensor0 * tensor1 + 0.5)", "{}", "{ {x:0}:1, {x:1}:1 }"); tester.assertEvaluates("0.0", "sum(tensor0 * tensor1 + 0.5)", "tensor(x{}):{}", "{ {x:0}:1, {x:1}:1 }"); + tester.assertEvaluates("1", + "reduce(join(tensor0, tensor1, f(x,y) (if(x > y, 1.0, 0.0))), sum, tag) == reduce(tensor0, count, tag)", + "tensor(tag{}):{{tag:tag1}:10, {tag:tag2}:20}", "{5}"); + tester.assertEvaluates("0", + "reduce(join(tensor0, tensor1, f(x,y) (if(x > y, 1.0, 0.0))), sum, tag) == reduce(tensor0, count, tag)", + "tensor(tag{}):{{tag:tag1}:10, {tag:tag2}:20}", "{15}"); + tester.assertEvaluates("0", + "reduce(join(tensor0, tensor1, f(x,y) (if(x > y, 1.0, 0.0))), sum, tag) == reduce(tensor0, count, tag)", + "tensor(tag{}):{{tag:tag1}:10, {tag:tag2}:20}", "{25}"); + // tensor result dimensions are given from argument dimensions, not the resulting values tester.assertEvaluates("tensor(x{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{}):{ {x:1}:1 }"); tester.assertEvaluates("tensor(x{},y{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{},y{}):{ {x:1,y:0}:1, {x:2,y:1}:1 }"); diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 6f374f1bea4..512252fd732 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -5,14 +5,13 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/vespalib/objects/objectdumper.h> #include <vespa/vespalib/testkit/testapp.h> -#include <stdexcept> #include <vespa/document/base/testdocman.h> #include <vespa/vespalib/util/md5.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h> #include <vespa/searchlib/expression/documentfieldnode.h> #include <cmath> #include <iostream> +#include <list> #define MU std::make_unique diff --git a/searchlib/src/tests/alignment/alignment.cpp b/searchlib/src/tests/alignment/alignment.cpp index 882e0942976..19ba0b4048b 100644 --- a/searchlib/src/tests/alignment/alignment.cpp +++ b/searchlib/src/tests/alignment/alignment.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("alignment_test"); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 421c3ce7138..44ce0498cf1 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/update/arithmeticvalueupdate.h> @@ -214,8 +213,7 @@ private: void testGeneration(const AttributePtr & attr, bool exactStatus); void testGeneration(); - void - testCreateSerialNum(void); + void testCreateSerialNum(); void testPredicateHeaderTags(); @@ -232,8 +230,7 @@ private: void testCompactLidSpace(const Config &config); - void - testCompactLidSpace(void); + void testCompactLidSpace(); template <typename AttributeType> void requireThatAddressSpaceUsageIsReported(const Config &config, bool fastSearch); diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index afba4287424..04a31c2dcfb 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("attributefilewriter_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/attribute/attributefilewriter.h> @@ -13,14 +10,16 @@ LOG_SETUP("attributefilewriter_test"); #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP("attributefilewriter_test"); using search::index::DummyFileHeaderContext; -namespace search -{ +namespace search { -namespace -{ +namespace { vespalib::string testFileName("test.dat"); vespalib::string hello("Hello world"); diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 1f53bbc3ea1..7247436845a 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); #include <vespa/searchlib/attribute/attribute.h> @@ -84,7 +83,7 @@ public: } void - updateFirstUsedGen(void) + updateFirstUsedGen() { updateFirstUsedGeneration(); } diff --git a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp index 9f30f1e0713..6a180317a9e 100644 --- a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp +++ b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> @@ -10,11 +9,13 @@ #include <vespa/searchlib/attribute/multistringattribute.h> #include <vespa/searchlib/attribute/attrvector.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/fastos/app.h> #include <iostream> #include <fstream> #include "../attributesearcher.h" #include "../attributeupdater.h" #include "../runnable.h" + #include <vespa/log/log.h> LOG_SETUP("attributebenchmark"); diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index 525643e377c..e08420e2521 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchlib/src/tests/attribute/comparator/comparator_test.cpp b/searchlib/src/tests/attribute/comparator/comparator_test.cpp index ccc565407b8..a3547088e41 100644 --- a/searchlib/src/tests/attribute/comparator/comparator_test.cpp +++ b/searchlib/src/tests/attribute/comparator/comparator_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("comparator_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 24a9d85a068..87307a08297 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -47,7 +47,7 @@ typedef std::unique_ptr<AttributeVector::SearchContext> SearchContextPtr; typedef std::unique_ptr<search::queryeval::SearchIterator> SearchBasePtr; bool -FastOS_UNIX_File::Sync(void) +FastOS_UNIX_File::Sync() { // LOG(info, "Skip sync"); return true; diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 7944e81ac55..2a32892634d 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("enumstore_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp index 9e15f666130..5b8a5966136 100644 --- a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp +++ b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("extendattribute_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/attribute/guard/attributeguard.cpp b/searchlib/src/tests/attribute/guard/attributeguard.cpp index 9eef5e93490..d3f9c607a60 100644 --- a/searchlib/src/tests/attribute/guard/attributeguard.cpp +++ b/searchlib/src/tests/attribute/guard/attributeguard.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attributeguard_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 35e702dcaf1..ed62f220bd2 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -36,7 +36,7 @@ private: int _value; uint32_t _order; - RandomValue(void) + RandomValue() : _docId(0), _value(0u), _order(0u) @@ -146,7 +146,7 @@ private: uint32_t _generation; void - allocTree(void); + allocTree(); void freeTree(bool verbose); @@ -180,7 +180,7 @@ private: RandomValuesVector &values); void - sortRandomValues(void); + sortRandomValues(); void doCompactEnumStore(Tree &tree, @@ -214,7 +214,7 @@ public: AttributePostingListTest(); ~AttributePostingListTest(); - int Main(void) override; + int Main() override; }; AttributePostingListTest::AttributePostingListTest() diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index b2bc047553e..5c0ab516b64 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -165,8 +165,8 @@ private: template <typename VectorType> void testMinMax(AttributePtr &ptr1, AttributePtr &ptr2); - void testMinMax(void); - void testStringFold(void); + void testMinMax(); + void testStringFold(); void testDupValuesInIntArray(); void testDupValuesInStringArray(); public: @@ -990,7 +990,7 @@ PostingListAttributeTest::testMinMax(AttributePtr &ptr1, AttributePtr &ptr2) } void -PostingListAttributeTest::testMinMax(void) +PostingListAttributeTest::testMinMax() { { Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); @@ -1026,7 +1026,7 @@ PostingListAttributeTest::testMinMax(void) void -PostingListAttributeTest::testStringFold(void) +PostingListAttributeTest::testStringFold() { Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index 25222706f53..5d219fe8918 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("reference_attribute_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/attribute/runnable.h b/searchlib/src/tests/attribute/runnable.h index e36af939943..6e5bd1eaf42 100644 --- a/searchlib/src/tests/attribute/runnable.h +++ b/searchlib/src/tests/attribute/runnable.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/vespalib/util/sync.h> +#include <vespa/fastos/thread.h> namespace search { diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index c9ebe399ab5..581dc61aabb 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 6c8d67c7a61..b341f2a29d0 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attributeblueprint_test"); diff --git a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp index 0babb633404..d7287ed8439 100644 --- a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp +++ b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for sourceselector. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("sourceselector_test"); - #include <vespa/searchlib/attribute/fixedsourceselector.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP("sourceselector_test"); using std::unique_ptr; using std::string; diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index f1034806e36..749a0192302 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("stringattribute_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp index 17e0641ccaf..789a216267f 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp +++ b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("expglomb_test"); #include <vespa/searchlib/bitcompression/compression.h> diff --git a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp index 400d16b3d86..0839d224780 100644 --- a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp +++ b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/searchlib/common/bitvector.h> +#include <vespa/fastos/app.h> #include <iostream> #include <string> #include <vector> diff --git a/searchlib/src/tests/btree/btreeaggregation_test.cpp b/searchlib/src/tests/btree/btreeaggregation_test.cpp index ae9cb5b21c7..45302c714fa 100644 --- a/searchlib/src/tests/btree/btreeaggregation_test.cpp +++ b/searchlib/src/tests/btree/btreeaggregation_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("btreeaggregation_test"); #include <vespa/vespalib/testkit/testapp.h> @@ -136,7 +135,7 @@ class WrapInt public: int _val; WrapInt(int val) : _val(val) {} - WrapInt(void) : _val(0) {} + WrapInt() : _val(0) {} bool operator==(const WrapInt & rhs) const { return _val == rhs._val; } }; @@ -1031,7 +1030,7 @@ Test::requireThatUpdateOfDataWorks() template <typename TreeStore> void -Test::requireThatSmallNodesWorks(void) +Test::requireThatSmallNodesWorks() { GenerationHandler g; TreeStore s; diff --git a/searchlib/src/tests/btree/iteratespeed.cpp b/searchlib/src/tests/btree/iteratespeed.cpp index bc1be5b495c..0a91b5a8a1b 100644 --- a/searchlib/src/tests/btree/iteratespeed.cpp +++ b/searchlib/src/tests/btree/iteratespeed.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("iteratespeed"); -#include <string> + #include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreebuilder.h> #include <vespa/searchlib/btree/btreenodeallocator.h> #include <vespa/searchlib/btree/btree.h> #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/util/rand48.h> - #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> @@ -19,6 +15,12 @@ LOG_SETUP("iteratespeed"); #include <vespa/searchlib/btree/btree.hpp> #include <vespa/searchlib/btree/btreestore.hpp> +#include <vespa/fastos/app.h> +#include <vespa/fastos/timestamp.h> + +#include <vespa/log/log.h> +LOG_SETUP("iteratespeed"); + namespace search { namespace btree { @@ -35,11 +37,8 @@ class IterateSpeed : public FastOS_Application void workLoop(int loops, bool enableForward, bool enableBackwards, bool enableLambda, int leafSlots); - void usage(); - - int - Main(void) override; + int Main() override; }; diff --git a/searchlib/src/tests/bytecomplens/bytecomp.cpp b/searchlib/src/tests/bytecomplens/bytecomp.cpp index 78b5cbd24dd..87293b43842 100644 --- a/searchlib/src/tests/bytecomplens/bytecomp.cpp +++ b/searchlib/src/tests/bytecomplens/bytecomp.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <memory> -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("bytecomplens_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/bytecomplens/tblprint.cpp b/searchlib/src/tests/bytecomplens/tblprint.cpp index 93657d82178..bef3f3db662 100644 --- a/searchlib/src/tests/bytecomplens/tblprint.cpp +++ b/searchlib/src/tests/bytecomplens/tblprint.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("tblprint"); #include <vespa/vespalib/util/random.h> diff --git a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp index cc0ef78c193..3c352aac04c 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_benchmark.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("bitvector_benchmark"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp index eddd3941c35..9786f53f921 100644 --- a/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/condensedbitvector_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/condensedbitvectors.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp b/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp index 49ebbf12bc0..dea9a8534f6 100644 --- a/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp +++ b/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("foregroundtaskexecutor_test"); #include <vespa/searchlib/common/foregroundtaskexecutor.h> diff --git a/searchlib/src/tests/common/resultset/resultset_test.cpp b/searchlib/src/tests/common/resultset/resultset_test.cpp index 983dc10b914..fcfc6299707 100644 --- a/searchlib/src/tests/common/resultset/resultset_test.cpp +++ b/searchlib/src/tests/common/resultset/resultset_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for resultset. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("resultset_test"); diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index e8183319fd4..12fc6241255 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("sequencedtaskexecutor_test"); #include <vespa/searchlib/common/sequencedtaskexecutor.h> diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp index 6d4e8bc49c8..955a5241a3d 100644 --- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp +++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("summaryfeatures_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/datastore/datastore/datastore_test.cpp b/searchlib/src/tests/datastore/datastore/datastore_test.cpp index 2c9b08f8a12..1da8a03e068 100644 --- a/searchlib/src/tests/datastore/datastore/datastore_test.cpp +++ b/searchlib/src/tests/datastore/datastore/datastore_test.cpp @@ -50,7 +50,7 @@ public: } void - switchActiveBuffer(void) + switchActiveBuffer() { ParentType::switchActiveBuffer(0, 0u); } @@ -140,7 +140,7 @@ private: void requireThatWeCanUseFreeLists(); void requireThatMemoryStatsAreCalculated(); void requireThatMemoryUsageIsCalculated(); - void requireThatWecanDisableElemHoldList(void); + void requireThatWecanDisableElemHoldList(); void requireThatBufferGrowthWorks(); public: int Main() override; @@ -442,7 +442,7 @@ Test::requireThatMemoryUsageIsCalculated() void -Test::requireThatWecanDisableElemHoldList(void) +Test::requireThatWecanDisableElemHoldList() { MyStore s; MyRef r1 = s.addEntry(10); diff --git a/searchlib/src/tests/datastore/unique_store/unique_store_test.cpp b/searchlib/src/tests/datastore/unique_store/unique_store_test.cpp index 16eaea9d04b..4064921aac5 100644 --- a/searchlib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/searchlib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("unique_store_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp index f3c2f73c78c..8252d0c622a 100644 --- a/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/diskindex/bitvector/bitvector_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("bitvector_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index 6bde0b965a6..4135e90b0e7 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -62,22 +62,19 @@ uint32_t minChunkDocs = 262144; vespalib::string dirprefix = "index/"; -void -disableSkip(void) +void disableSkip() { minSkipDocs = 10000000; minChunkDocs = 1 << 30; } -void -enableSkip(void) +void enableSkip() { minSkipDocs = 64; minChunkDocs = 1 << 30; } -void -enableSkipChunks(void) +void enableSkipChunks() { minSkipDocs = 64; minChunkDocs = 9000; // Unrealistic low for testing @@ -110,17 +107,17 @@ public: search::Rand48 _rnd; private: - void Usage(void); + void Usage(); void testFake(const std::string &postingType, FakeWord &fw); public: - FieldWriterTest(void); - ~FieldWriterTest(void); - int Main(void) override; + FieldWriterTest(); + ~FieldWriterTest(); + int Main() override; }; void -FieldWriterTest::Usage(void) +FieldWriterTest::Usage() { printf("fieldwriter_test " "[-c <commonDocFreq>] " @@ -130,7 +127,7 @@ FieldWriterTest::Usage(void) } -FieldWriterTest::FieldWriterTest(void) +FieldWriterTest::FieldWriterTest() : _verbose(false), _numDocs(3000000), _commonDocFreq(50000), @@ -142,7 +139,7 @@ FieldWriterTest::FieldWriterTest(void) } -FieldWriterTest::~FieldWriterTest(void) +FieldWriterTest::~FieldWriterTest() { } @@ -197,7 +194,7 @@ WrappedFieldWriter::WrappedFieldWriter(const vespalib::string &namepref, void -WrappedFieldWriter::earlyOpen(void) +WrappedFieldWriter::earlyOpen() { TuneFileSeqWrite tuneFileWrite; _fieldWriter.reset(new FieldWriter(_docIdLimit, _numWordIds)); @@ -209,7 +206,7 @@ WrappedFieldWriter::earlyOpen(void) void -WrappedFieldWriter::lateOpen(void) +WrappedFieldWriter::lateOpen() { TuneFileSeqWrite tuneFileWrite; DummyFileHeaderContext fileHeaderContext; @@ -219,7 +216,7 @@ WrappedFieldWriter::lateOpen(void) void -WrappedFieldWriter::open(void) +WrappedFieldWriter::open() { earlyOpen(); lateOpen(); @@ -227,7 +224,7 @@ WrappedFieldWriter::open(void) void -WrappedFieldWriter::close(void) +WrappedFieldWriter::close() { _fieldWriter->close(); _fieldWriter.reset(); @@ -235,7 +232,7 @@ WrappedFieldWriter::close(void) void -WrappedFieldWriter::writeCheckPoint(void) +WrappedFieldWriter::writeCheckPoint() { CheckPointFile chkptfile("chkpt"); nbostream out; @@ -260,7 +257,7 @@ WrappedFieldWriter::readCheckPoint(bool first) void -WrappedFieldWriter::checkPoint(void) +WrappedFieldWriter::checkPoint() { writeCheckPoint(); _fieldWriter.reset(); @@ -288,28 +285,14 @@ public: uint32_t numWordIds, uint32_t docIdLimit); - ~WrappedFieldReader(void); - - void - earlyOpen(void); - - void - lateOpen(void); - - void - open(void); - - void - close(void); - - void - writeCheckPoint(void); - - void - readCheckPoint(bool first); - - virtual void - checkPoint(void) override; + ~WrappedFieldReader(); + void earlyOpen(); + void lateOpen(); + void open(); + void close(); + void writeCheckPoint(); + void readCheckPoint(bool first); + virtual void checkPoint() override; }; @@ -336,13 +319,13 @@ WrappedFieldReader::WrappedFieldReader(const vespalib::string &namepref, } -WrappedFieldReader::~WrappedFieldReader(void) +WrappedFieldReader::~WrappedFieldReader() { } void -WrappedFieldReader::earlyOpen(void) +WrappedFieldReader::earlyOpen() { TuneFileSeqRead tuneFileRead; _fieldReader.reset(new FieldReader()); @@ -351,7 +334,7 @@ WrappedFieldReader::earlyOpen(void) void -WrappedFieldReader::lateOpen(void) +WrappedFieldReader::lateOpen() { TuneFileSeqRead tuneFileRead; _wmap.setup(_numWordIds); @@ -362,7 +345,7 @@ WrappedFieldReader::lateOpen(void) void -WrappedFieldReader::open(void) +WrappedFieldReader::open() { earlyOpen(); lateOpen(); @@ -370,7 +353,7 @@ WrappedFieldReader::open(void) void -WrappedFieldReader::close(void) +WrappedFieldReader::close() { _fieldReader->close(); _fieldReader.reset(); @@ -378,7 +361,7 @@ WrappedFieldReader::close(void) void -WrappedFieldReader::writeCheckPoint(void) +WrappedFieldReader::writeCheckPoint() { CheckPointFile chkptfile("chkpt"); nbostream out; @@ -403,7 +386,7 @@ WrappedFieldReader::readCheckPoint(bool first) void -WrappedFieldReader::checkPoint(void) +WrappedFieldReader::checkPoint() { writeCheckPoint(); _fieldReader.reset(); @@ -898,7 +881,7 @@ testFieldWriterVariantsWithHighLids(FakeWordSet &wordSet, uint32_t docIdLimit, } int -FieldWriterTest::Main(void) +FieldWriterTest::Main() { int argi; char c; diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index 2d12b5c1782..b96b2b4ff05 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/bitcompression/compression.h> -#include <vector> #include <vespa/searchlib/util/rand48.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/bitcompression/countcompression.h> @@ -13,6 +11,9 @@ #include <vespa/searchlib/diskindex/pagedict4file.h> #include <vespa/searchlib/diskindex/pagedict4randread.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/fastos/app.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP("pagedict4test"); @@ -65,15 +66,14 @@ public: { } - ~Writer(void) + ~Writer() { delete _ssw; delete _spw; delete _pw; } - void - allocWriters() + void allocWriters() { _ssw = new PageDict4SSWriter(_sse); _spw = new PageDict4SPWriter(*_ssw, _spe); @@ -82,16 +82,14 @@ public: _pw->setup(); } - void - flush(void) + void flush() { _pw->flush(); ThreeLevelCountWriteBuffers::flush(); } - void - addCounts(const std::string &word, - const PostingListCounts &counts) + void addCounts(const std::string &word, + const PostingListCounts &counts) { _pw->addCounts(word, counts); } @@ -119,10 +117,9 @@ public: _pr.setup(); } - void - readCounts(vespalib::string &word, - uint64_t &wordNum, - PostingListCounts &counts) + void readCounts(vespalib::string &word, + uint64_t &wordNum, + PostingListCounts &counts) { _pr.readCounts(word, wordNum, counts); } @@ -215,16 +212,10 @@ public: bool _firstWordForcedCommon; bool _lastWordForcedCommon; - void - usage(void); - - int - Main(void) override; - - void - testWords(void); - - PageDict4TestApp(void) + void usage(); + int Main() override; + void testWords(); + PageDict4TestApp() : _rnd(), _stress(false), _emptyWord(false), @@ -236,7 +227,7 @@ public: void -PageDict4TestApp::usage(void) +PageDict4TestApp::usage() { printf("Usage: wordnumbers\n"); fflush(stdout); @@ -244,7 +235,7 @@ PageDict4TestApp::usage(void) int -PageDict4TestApp::Main(void) +PageDict4TestApp::Main() { if (_argc > 0) { DummyFileHeaderContext::setCreator(_argv[0]); @@ -836,7 +827,7 @@ testWords(const std::string &logname, void -PageDict4TestApp::testWords(void) +PageDict4TestApp::testWords() { ::testWords("smallchunkwordsempty", _rnd, 1000000, 0, diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index 45637828ab1..b2b73b1d9ef 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -28,7 +28,7 @@ struct NullDataStore : IDataStore { void accept(IDataStoreVisitor &, IDataStoreVisitorProgress &, bool) override { } double getVisitCost() const override { return 1.0; } virtual DataStoreStorageStats getStorageStats() const override { - return DataStoreStorageStats(0, 0, 0.0, 0, 0); + return DataStoreStorageStats(0, 0, 0.0, 0, 0, 0); } virtual MemoryUsage getMemoryUsage() const override { return MemoryUsage(); } virtual std::vector<DataStoreFileChunkStats> @@ -38,6 +38,7 @@ struct NullDataStore : IDataStore { } virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } virtual bool canShrinkLidSpace() const override { return false; } + virtual size_t getEstimatedShrinkLidSpaceGain() const override { return 0; } virtual void shrinkLidSpace() override {} }; diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 649c230ba5c..df1fffa2a0e 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -35,7 +36,7 @@ const string header_name = doc_type_name + ".header"; const string body_name = doc_type_name + ".body"; document::DocumenttypesConfig -makeDocTypeRepoConfig(void) +makeDocTypeRepoConfig() { const int32_t doc_type_id = 787121340; document::config_builder::DocumenttypesConfigBuilderHelper builder; @@ -79,7 +80,7 @@ class MyTlSyncer : public transactionlog::SyncProxy SerialNum _syncedTo; public: - MyTlSyncer(void) + MyTlSyncer() : _syncedTo(0) { } diff --git a/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt new file mode 100644 index 00000000000..66b618d2885 --- /dev/null +++ b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_file_chunk_test_app TEST + SOURCES + file_chunk_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_file_chunk_test_app COMMAND searchlib_file_chunk_test_app) diff --git a/searchlib/src/tests/docstore/file_chunk/FILES b/searchlib/src/tests/docstore/file_chunk/FILES new file mode 100644 index 00000000000..9425cdb4f00 --- /dev/null +++ b/searchlib/src/tests/docstore/file_chunk/FILES @@ -0,0 +1 @@ +file_chunk_test.cpp diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp new file mode 100644 index 00000000000..824249b0a32 --- /dev/null +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -0,0 +1,213 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> + +#include <vespa/searchlib/common/fileheadercontext.h> +#include <vespa/searchlib/docstore/filechunk.h> +#include <vespa/searchlib/docstore/writeablefilechunk.h> +#include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <iomanip> +#include <iostream> + +#include <vespa/log/log.h> +LOG_SETUP("file_chunk_test"); + +using namespace search; + +using common::FileHeaderContext; +using vespalib::ThreadStackExecutor; + +struct MyFileHeaderContext : public FileHeaderContext { + virtual void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const override { + (void) header; + (void) name; + } +}; + +struct SetLidObserver : public ISetLid { + std::vector<uint32_t> lids; + virtual void setLid(const vespalib::LockGuard &guard, uint32_t lid, const LidInfo &lidInfo) override { + (void) guard; + (void) lidInfo; + lids.push_back(lid); + } +}; + +struct BucketizerObserver : public IBucketizer { + mutable std::vector<uint32_t> lids; + virtual document::BucketId getBucketOf(const vespalib::GenerationHandler::Guard &guard, uint32_t lid) const override { + (void) guard; + lids.push_back(lid); + return document::BucketId(); + } + virtual vespalib::GenerationHandler::Guard getGuard() const override { + return vespalib::GenerationHandler::Guard(); + } +}; + +vespalib::string +getData(uint32_t lid) +{ + std::ostringstream oss; + oss << "data_" << std::setw(5) << std::setfill('0') << lid; + return oss.str(); +} + +struct FixtureBase { + test::DirectoryHandler dir; + ThreadStackExecutor executor; + uint64_t serialNum; + TuneFileSummary tuneFile; + MyFileHeaderContext fileHeaderCtx; + vespalib::Lock updateLock; + SetLidObserver lidObserver; + BucketizerObserver bucketizer; + + uint64_t nextSerialNum() { + return serialNum++; + }; + + FixtureBase(const vespalib::string &baseName, + bool dirCleanup = true) + : dir(baseName), + executor(1, 0x10000), + serialNum(1), + tuneFile(), + fileHeaderCtx(), + updateLock(), + lidObserver(), + bucketizer() + { + dir.cleanup(dirCleanup); + } + ~FixtureBase() {} + void assertLidMap(const std::vector<uint32_t> &expLids) { + EXPECT_EQUAL(expLids, lidObserver.lids); + } + void assertBucketizer(const std::vector<uint32_t> &expLids) { + EXPECT_EQUAL(expLids, bucketizer.lids); + } +}; + +struct ReadFixture : public FixtureBase { + FileChunk chunk; + + ReadFixture(const vespalib::string &baseName, + bool dirCleanup = true) + : FixtureBase(baseName, dirCleanup), + chunk(FileChunk::FileId(0), + FileChunk::NameId(1234), + baseName, + tuneFile, + &bucketizer, + false) + { + dir.cleanup(dirCleanup); + } + void updateLidMap(uint32_t docIdLimit) { + vespalib::LockGuard guard(updateLock); + chunk.updateLidMap(guard, lidObserver, serialNum, docIdLimit); + } +}; + +struct WriteFixture : public FixtureBase { + WriteableFileChunk chunk; + + WriteFixture(const vespalib::string &baseName, + uint32_t docIdLimit, + bool dirCleanup = true) + : FixtureBase(baseName, dirCleanup), + chunk(executor, + FileChunk::FileId(0), + FileChunk::NameId(1234), + baseName, + serialNum, + docIdLimit, + WriteableFileChunk::Config(document::CompressionConfig(), 0x1000), + tuneFile, + fileHeaderCtx, + &bucketizer, + false) + { + dir.cleanup(dirCleanup); + } + void flush() { + chunk.flush(true, serialNum); + chunk.flushPendingChunks(serialNum); + } + WriteFixture &append(uint32_t lid) { + vespalib::string data = getData(lid); + chunk.append(nextSerialNum(), lid, data.c_str(), data.size()); + return *this; + } + void updateLidMap(uint32_t docIdLimit) { + vespalib::LockGuard guard(updateLock); + chunk.updateLidMap(guard, lidObserver, serialNum, docIdLimit); + } + +}; + +TEST_F("require that idx file without docIdLimit in header can be read by FileChunk", + ReadFixture("without_doc_id_limit", false)) +{ + EXPECT_EQUAL(std::numeric_limits<uint32_t>::max(), f.chunk.getDocIdLimit()); +} + +TEST_F("require that idx file without docIdLimit in header can be read by WriteableFileChunk", + WriteFixture("without_doc_id_limit", 1000, false)) +{ + EXPECT_EQUAL(std::numeric_limits<uint32_t>::max(), f.chunk.getDocIdLimit()); +} + +TEST("require that docIdLimit is written to and read from idx file header") +{ + { + WriteFixture f("tmp", 1000, false); + EXPECT_EQUAL(1000u, f.chunk.getDocIdLimit()); + } + { + ReadFixture f("tmp", false); + f.updateLidMap(std::numeric_limits<uint32_t>::max()); // trigger reading of idx file header + EXPECT_EQUAL(1000u, f.chunk.getDocIdLimit()); + } + { + WriteFixture f("tmp", 0); + EXPECT_EQUAL(1000u, f.chunk.getDocIdLimit()); + } +} + +template <typename FixtureType> +void +assertUpdateLidMap(FixtureType &f) +{ + std::vector<uint32_t> expLids({1,10,100,999,998,999}); + f.assertLidMap(expLids); + f.assertBucketizer(expLids); + size_t entrySize = 10 + 8; + EXPECT_EQUAL(9 * entrySize, f.chunk.getAddedBytes()); + EXPECT_EQUAL(3u, f.chunk.getBloatCount()); + EXPECT_EQUAL(3 * entrySize, f.chunk.getErasedBytes()); +} + +TEST("require that entries with lid >= docIdLimit are skipped in updateLidMap()") +{ + { + WriteFixture f("tmp", 0, false); + f.append(1).append(10).append(100).append(999).append(1000).append(1001).append(998).append(1002).append(999); + f.flush(); + } + { + ReadFixture f("tmp", false); + f.updateLidMap(1000); + assertUpdateLidMap(f); + } + { + WriteFixture f("tmp", 0); + f.updateLidMap(1000); + assertUpdateLidMap(f); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } + diff --git a/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.dat b/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.dat Binary files differnew file mode 100644 index 00000000000..1b8edd69fe8 --- /dev/null +++ b/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.dat diff --git a/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.idx b/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.idx Binary files differnew file mode 100644 index 00000000000..f2fda58f94e --- /dev/null +++ b/searchlib/src/tests/docstore/file_chunk/without_doc_id_limit/0000000000000001234.idx diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index aa7cf03be31..46e7333ebb7 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -1,16 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/searchlib/docstore/logdocumentstore.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/searchlib/docstore/chunkformats.h> +#include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/docstore/storebybucket.h> #include <vespa/searchlib/docstore/visitcache.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/vespalib/util/exceptions.h> +#include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <iomanip> +#include <iostream> using document::BucketId; using namespace search::docstore; @@ -21,7 +26,7 @@ using search::index::DummyFileHeaderContext; class MyTlSyncer : public transactionlog::SyncProxy { SerialNum _syncedTo; public: - MyTlSyncer(void) : _syncedTo(0) { } + MyTlSyncer() : _syncedTo(0) { } void sync(SerialNum syncTo) override { _syncedTo = syncTo; @@ -729,6 +734,224 @@ TEST("testBucketDensityComputer") { EXPECT_EQUAL(0u, nonRecording.getNumBuckets()); } +LogDataStore::Config +getBasicConfig(size_t maxFileSize) +{ + CompressionConfig compCfg; + WriteableFileChunk::Config fileCfg; + return LogDataStore::Config(maxFileSize, 0.2, 2.5, 0.2, 1, true, compCfg, fileCfg); +} + +vespalib::string +genData(uint32_t lid, size_t numBytes) +{ + assert(numBytes >= 6); + std::ostringstream oss; + for (size_t i = 0; i < (numBytes - 6); ++i) { + oss << 'a'; + } + oss << std::setw(6) << std::setfill('0') << lid; + return oss.str(); +} + +struct Fixture { + vespalib::ThreadStackExecutor executor; + search::test::DirectoryHandler dir; + uint64_t serialNum; + DummyFileHeaderContext fileHeaderCtx; + MyTlSyncer tlSyncer; + LogDataStore store; + + uint64_t nextSerialNum() { + return serialNum++; + } + + Fixture(const vespalib::string &dirName = "tmp", + bool dirCleanup = true, + size_t maxFileSize = 4096 * 2) + : executor(1, 0x10000), + dir(dirName), + serialNum(0), + fileHeaderCtx(), + tlSyncer(), + store(executor, + dirName, + getBasicConfig(maxFileSize), + GrowStrategy(), + TuneFileSummary(), + fileHeaderCtx, + tlSyncer, + nullptr) + { + dir.cleanup(dirCleanup); + } + ~Fixture() {} + void flush() { + store.initFlush(serialNum); + store.flush(serialNum); + } + Fixture &write(uint32_t lid, size_t numBytes = 1024) { + vespalib::string data = genData(lid, numBytes); + store.write(nextSerialNum(), lid, data.c_str(), data.size()); + return *this; + } + uint32_t writeUntilNewChunk(uint32_t startLid) { + size_t numChunksStart = store.getFileChunkStats().size(); + for (uint32_t lid = startLid; ; ++lid) { + write(lid); + if (store.getFileChunkStats().size() > numChunksStart) { + return lid; + } + } + } + void compactLidSpace(uint32_t wantedDocIdLimit) { + store.compactLidSpace(wantedDocIdLimit); + assertDocIdLimit(wantedDocIdLimit); + } + void assertDocIdLimit(uint32_t expDocIdLimit) { + EXPECT_EQUAL(expDocIdLimit, store.getDocIdLimit()); + } + void assertNumChunks(size_t numChunks) { + EXPECT_EQUAL(numChunks, store.getFileChunkStats().size()); + } + void assertDocIdLimitInFileChunks(const std::vector<uint32_t> expLimits) { + std::vector<uint32_t> actLimits; + for (const auto &stat : store.getFileChunkStats()) { + actLimits.push_back(stat.docIdLimit()); + } + EXPECT_EQUAL(expLimits, actLimits); + } + void assertContent(const std::set<uint32_t> &lids, uint32_t docIdLimit, size_t numBytesPerEntry = 1024) { + for (uint32_t lid = 0; lid < docIdLimit; ++lid) { + vespalib::DataBuffer buffer; + size_t size = store.read(lid, buffer); + if (lids.find(lid) != lids.end()) { + vespalib::string expData = genData(lid, numBytesPerEntry); + EXPECT_EQUAL(expData, vespalib::string(buffer.getData(), buffer.getDataLen())); + EXPECT_GREATER(size, 0u); + } else { + EXPECT_EQUAL("", vespalib::string(buffer.getData(), buffer.getDataLen())); + EXPECT_EQUAL(0u, size); + } + } + } +}; + +TEST("require that docIdLimit is updated when inserting entries") +{ + { + Fixture f("tmp", false); + f.assertDocIdLimit(0); + f.write(10); + f.assertDocIdLimit(11); + f.write(9); + f.assertDocIdLimit(11); + f.write(11); + f.assertDocIdLimit(12); + f.assertNumChunks(1); + f.flush(); + } + { + Fixture f("tmp"); + f.assertDocIdLimit(12); + } +} + +TEST("require that docIdLimit at idx file creation time is written to idx file header") +{ + std::vector<uint32_t> expLimits = {std::numeric_limits<uint32_t>::max(),14,104,204}; + { + Fixture f("tmp", false); + f.writeUntilNewChunk(10); + f.writeUntilNewChunk(100); + f.writeUntilNewChunk(200); + f.assertDocIdLimitInFileChunks(expLimits); + f.flush(); + } + { + Fixture f("tmp"); + f.assertDocIdLimitInFileChunks(expLimits); + } +} + +TEST("require that lid space can be compacted and entries from old files skipped during load") +{ + { + Fixture f("tmp", false); + f.write(10); + f.writeUntilNewChunk(100); + f.write(20); + f.writeUntilNewChunk(200); + f.write(30); + TEST_DO(f.assertContent({10,100,101,102,20,200,201,202,30}, 203)); + + f.assertDocIdLimit(203); + f.compactLidSpace(100); + TEST_DO(f.assertContent({10,20,30}, 203)); + + f.writeUntilNewChunk(31); + f.write(99); + f.write(300); + TEST_DO(f.assertContent({10,20,30,31,32,33,99,300}, 301)); + f.assertDocIdLimitInFileChunks({std::numeric_limits<uint32_t>::max(),103,203,100}); + f.flush(); + } + { + Fixture f("tmp"); + TEST_DO(f.assertContent({10,20,30,31,32,33,99,300}, 301)); + } +} + +TEST_F("require that getLid() is protected by docIdLimit", Fixture) +{ + f.write(1); + vespalib::GenerationHandler::Guard guard = f.store.getLidReadGuard(); + EXPECT_TRUE(f.store.getLid(guard, 1).valid()); + EXPECT_FALSE(f.store.getLid(guard, 2).valid()); +} + +TEST_F("require that lid space can be compacted and shrunk", Fixture) +{ + f.write(1).write(2); + EXPECT_FALSE(f.store.canShrinkLidSpace()); + + f.compactLidSpace(2); + MemoryUsage before = f.store.getMemoryUsage(); + EXPECT_TRUE(f.store.canShrinkLidSpace()); + EXPECT_EQUAL(8u, f.store.getEstimatedShrinkLidSpaceGain()); // one lid info entry + f.store.shrinkLidSpace(); + + MemoryUsage after = f.store.getMemoryUsage(); + EXPECT_LESS(after.usedBytes(), before.usedBytes()); + EXPECT_EQUAL(8u, before.usedBytes() - after.usedBytes()); +} + +TEST_F("require that lid space can be increased after being compacted and then shrunk", Fixture) +{ + f.write(1).write(3); + TEST_DO(f.compactLidSpace(2)); + f.write(2); + TEST_DO(f.assertDocIdLimit(3)); + f.store.shrinkLidSpace(); + TEST_DO(f.assertDocIdLimit(3)); + TEST_DO(f.assertContent({1,2}, 3)); +} + +TEST_F("require that lid space can be shrunk only after read guards are deleted", Fixture) +{ + f.write(1).write(2); + EXPECT_FALSE(f.store.canShrinkLidSpace()); + { + vespalib::GenerationHandler::Guard guard = f.store.getLidReadGuard(); + f.compactLidSpace(2); + f.write(1); // trigger remove of old generations + EXPECT_FALSE(f.store.canShrinkLidSpace()); + EXPECT_EQUAL(0u, f.store.getEstimatedShrinkLidSpaceGain()); + } + f.write(1); // trigger remove of old generations + EXPECT_TRUE(f.store.canShrinkLidSpace()); + EXPECT_EQUAL(8u, f.store.getEstimatedShrinkLidSpaceGain()); +} TEST_MAIN() { DummyFileHeaderContext::setCreator("logdatastore_test"); diff --git a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp b/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp index b89bf6f093f..0d99a065ce1 100644 --- a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp +++ b/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("docsumapi_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp b/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp index 9198a4462c6..7214bffc454 100644 --- a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp +++ b/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("monitorapi_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp index 9cfc0d15678..188228ea96b 100644 --- a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp +++ b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("searchapi_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp index b7932ea2381..d8fd8a52b0e 100644 --- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp +++ b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/engine/transportserver.h> - +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/fnet.h> #include <vespa/searchlib/engine/errorcodes.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp index 34b10d19715..16670c0ff30 100644 --- a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp +++ b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -8,6 +7,7 @@ #include <vespa/searchlib/features/element_completeness_feature.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace search::fef; using namespace search::fef::test; diff --git a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp index 7fa9e7c8bc2..f9e1fc902c8 100644 --- a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp +++ b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -9,6 +8,7 @@ #include <vespa/searchlib/fef/test/ftlib.h> #include <initializer_list> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace search::fef; using namespace search::fef::test; diff --git a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp index 00aa4ed7b20..7fc4cb4fbe8 100644 --- a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp +++ b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/attributefactory.h> diff --git a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp index 51405064ac6..44f4fd46765 100644 --- a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp +++ b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp index 89d59406b55..a84e8f42847 100644 --- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp +++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -9,6 +8,7 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/query/weight.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 770473d5308..004497411a2 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("prod_features_test"); diff --git a/searchlib/src/tests/features/prod_features_attributematch.cpp b/searchlib/src/tests/features/prod_features_attributematch.cpp index 724b8171b4e..e6e60aada96 100644 --- a/searchlib/src/tests/features/prod_features_attributematch.cpp +++ b/searchlib/src/tests/features/prod_features_attributematch.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP(".prod_features_attributematch"); diff --git a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp index 886feffefc4..3a3dba039ff 100644 --- a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp +++ b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".prod_features_fieldtermmatch"); #include "prod_features.h" #include <vespa/searchlib/features/fieldtermmatchfeature.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> +LOG_SETUP(".prod_features_fieldtermmatch"); using namespace search::features; using namespace search::fef; diff --git a/searchlib/src/tests/features/prod_features_framework.cpp b/searchlib/src/tests/features/prod_features_framework.cpp index 19a048063c3..3f70245bd79 100644 --- a/searchlib/src/tests/features/prod_features_framework.cpp +++ b/searchlib/src/tests/features/prod_features_framework.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP(".prod_features_framework"); diff --git a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp index 70e1d481a6c..b5d8ca58e10 100644 --- a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp +++ b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/value_type.h> diff --git a/searchlib/src/tests/features/raw_score/raw_score_test.cpp b/searchlib/src/tests/features/raw_score/raw_score_test.cpp index 1588e4dbe9f..acf5066b2c3 100644 --- a/searchlib/src/tests/features/raw_score/raw_score_test.cpp +++ b/searchlib/src/tests/features/raw_score/raw_score_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -8,6 +7,7 @@ #include <vespa/searchlib/features/raw_score_feature.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/subqueries/subqueries_test.cpp b/searchlib/src/tests/features/subqueries/subqueries_test.cpp index 71b9e8fc8db..0dfe2858735 100644 --- a/searchlib/src/tests/features/subqueries/subqueries_test.cpp +++ b/searchlib/src/tests/features/subqueries/subqueries_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -8,6 +7,7 @@ #include <vespa/searchlib/features/subqueries_feature.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp index a3e5808bffb..bf5416291d7 100644 --- a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp +++ b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -9,6 +8,7 @@ #include <vespa/searchlib/fef/test/ftlib.h> #include <initializer_list> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace search::fef; using namespace search::fef::test; diff --git a/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp index 70708536345..522a32f66d5 100644 --- a/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp +++ b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("attributecontent_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp index 0e574c776b5..cdd0ef24e51 100644 --- a/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp +++ b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("featurenamebuilder_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp b/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp index ef7d00f7cd8..64247790a85 100644 --- a/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp +++ b/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("featurenameparser_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp index ec82adc1b3a..c3f9fcb6063 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("featureoverride_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp index 2a3626ee834..841a5538f4c 100644 --- a/searchlib/src/tests/fef/fef_test.cpp +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("fef_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/parameter/parameter_test.cpp b/searchlib/src/tests/fef/parameter/parameter_test.cpp index 6efc5107f1e..ca8c7315ca1 100644 --- a/searchlib/src/tests/fef/parameter/parameter_test.cpp +++ b/searchlib/src/tests/fef/parameter/parameter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("parameter_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp index 7d868f96f2a..e77c17b2fbe 100644 --- a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("phrasesplitter_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp index d8721a5d682..ad61915949f 100644 --- a/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("phrasesplitter_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index c7400d5815b..bd6818cf6e4 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -216,11 +216,15 @@ TEST("test stuff") { { // test index properties known by the framework { // vespa.eval.lazy_expressions EXPECT_EQUAL(eval::LazyExpressions::NAME, vespalib::string("vespa.eval.lazy_expressions")); - EXPECT_EQUAL(eval::LazyExpressions::DEFAULT_VALUE, vespalib::string("false")); Properties p; - EXPECT_TRUE(!eval::LazyExpressions::check(p)); - p.add("vespa.eval.lazy_expressions", "true"); - EXPECT_TRUE(eval::LazyExpressions::check(p)); + EXPECT_TRUE(eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); + p = Properties().add("vespa.eval.lazy_expressions", "true"); + EXPECT_TRUE(eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(eval::LazyExpressions::check(p, false)); + p = Properties().add("vespa.eval.lazy_expressions", "false"); + EXPECT_TRUE(!eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); } { // vespa.rank.firstphase EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); diff --git a/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp index 2ce802ea8a2..8616553b994 100644 --- a/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp +++ b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("termmatchdatamerger_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fileheaderinspect/CMakeLists.txt b/searchlib/src/tests/fileheaderinspect/CMakeLists.txt deleted file mode 100644 index 4bc15f90ea7..00000000000 --- a/searchlib/src/tests/fileheaderinspect/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_fileheaderinspect_test_app TEST - SOURCES - fileheaderinspect.cpp - DEPENDS - searchlib - AFTER - searchlib_vespa-header-inspect_app -) -vespa_add_test(NAME searchlib_fileheaderinspect_test_app COMMAND searchlib_fileheaderinspect_test_app) diff --git a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp index eb0bd9e32b6..e30413ede90 100644 --- a/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp +++ b/searchlib/src/tests/fileheadertk/fileheadertk_test.cpp @@ -1,10 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("fileheadertk_test"); #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP("fileheadertk_test"); using namespace search; diff --git a/searchlib/src/tests/forcelink/forcelink.cpp b/searchlib/src/tests/forcelink/forcelink.cpp index 9f555e09480..2dc3291f1a1 100644 --- a/searchlib/src/tests/forcelink/forcelink.cpp +++ b/searchlib/src/tests/forcelink/forcelink.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("forcelink_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/grouping/hyperloglog_test.cpp b/searchlib/src/tests/grouping/hyperloglog_test.cpp index 15b4ae9ae39..6501d0633a5 100644 --- a/searchlib/src/tests/grouping/hyperloglog_test.cpp +++ b/searchlib/src/tests/grouping/hyperloglog_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("hyperloglog_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/grouping/hyperloglog.h> #include <vespa/vespalib/objects/nboserializer.h> diff --git a/searchlib/src/tests/grouping/sketch_test.cpp b/searchlib/src/tests/grouping/sketch_test.cpp index c6c0b144983..192f4998e5a 100644 --- a/searchlib/src/tests/grouping/sketch_test.cpp +++ b/searchlib/src/tests/grouping/sketch_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("sketch_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/grouping/sketch.h> #include <vespa/vespalib/objects/nboserializer.h> diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index 1a5f230a0c1..a3fdc924b32 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("hitcollector_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp index 63b1b78339e..f5eb6d385d9 100644 --- a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp +++ b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp @@ -9,7 +9,6 @@ * */ -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("docbuilder_test"); #include <boost/algorithm/string/classification.hpp> @@ -125,13 +124,13 @@ Test::testBuilder() endField(); b.startIndexField("iu"). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4"). + addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("81"). @@ -149,13 +148,13 @@ Test::testBuilder() b.startIndexField("iau"). startElement(1). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8"). + addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("82"). @@ -197,13 +196,13 @@ Test::testBuilder() b.startIndexField("iwu"). startElement(4). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12"). + addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("83"). @@ -310,8 +309,8 @@ Test::testBuilder() EXPECT_EQUAL("<item weight=\"1\">foo</item>", *itr++); EXPECT_EQUAL("</ic>", *itr++); EXPECT_EQUAL("<iu>", *itr++); - EXPECT_EQUAL("<all>http://www.yahoo.com:81/fluke?ab=2#4</all>", *itr++); - EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++); + EXPECT_EQUAL("<all>http://www.example.com:81/fluke?ab=2#4</all>", *itr++); + EXPECT_EQUAL("<host>www.example.com</host>", *itr++); EXPECT_EQUAL("<scheme>http</scheme>", *itr++); EXPECT_EQUAL("<path>/fluke</path>", *itr++); EXPECT_EQUAL("<port>81</port>", *itr++); @@ -320,8 +319,8 @@ Test::testBuilder() EXPECT_EQUAL("</iu>", *itr++); EXPECT_EQUAL("<iau>", *itr++); EXPECT_EQUAL("<item>", *itr++); - EXPECT_EQUAL("<all>http://www.yahoo.com:82/fluke?ab=2#8</all>", *itr++); - EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++); + EXPECT_EQUAL("<all>http://www.example.com:82/fluke?ab=2#8</all>", *itr++); + EXPECT_EQUAL("<host>www.example.com</host>", *itr++); EXPECT_EQUAL("<scheme>http</scheme>", *itr++); EXPECT_EQUAL("<path>/fluke</path>", *itr++); EXPECT_EQUAL("<port>82</port>", *itr++); @@ -340,8 +339,8 @@ Test::testBuilder() EXPECT_EQUAL("</iau>", *itr++); EXPECT_EQUAL("<iwu>", *itr++); EXPECT_EQUAL("<item weight=\"4\">", *itr++); - EXPECT_EQUAL("<all>http://www.yahoo.com:83/fluke?ab=2#12</all>", *itr++); - EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++); + EXPECT_EQUAL("<all>http://www.example.com:83/fluke?ab=2#12</all>", *itr++); + EXPECT_EQUAL("<host>www.example.com</host>", *itr++); EXPECT_EQUAL("<scheme>http</scheme>", *itr++); EXPECT_EQUAL("<path>/fluke</path>", *itr++); EXPECT_EQUAL("<port>83</port>", *itr++); diff --git a/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp b/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp index 8dd4afbc8d7..be3fd3cb601 100644 --- a/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp +++ b/searchlib/src/tests/index/doctypebuilder/doctypebuilder_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchlib/index/doctypebuilder.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/ld-library-path/ld-library-path.cpp b/searchlib/src/tests/ld-library-path/ld-library-path.cpp index c9a429b3b35..9ce019eb075 100644 --- a/searchlib/src/tests/ld-library-path/ld-library-path.cpp +++ b/searchlib/src/tests/ld-library-path/ld-library-path.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP(""); diff --git a/searchlib/src/tests/memoryindex/btree/btree_test.cpp b/searchlib/src/tests/memoryindex/btree/btree_test.cpp index b3f49a68424..7b88c2fb66e 100644 --- a/searchlib/src/tests/memoryindex/btree/btree_test.cpp +++ b/searchlib/src/tests/memoryindex/btree/btree_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("btree_test"); #include <vespa/vespalib/testkit/testapp.h> @@ -51,7 +50,7 @@ class WrapInt public: int _val; WrapInt(int val) : _val(val) {} - WrapInt(void) : _val(0) {} + WrapInt() : _val(0) {} bool operator==(const WrapInt & rhs) const { return _val == rhs._val; } }; @@ -944,7 +943,7 @@ Test::requireThatUpdateOfKeyWorks() void -Test::requireThatSmallNodesWorks(void) +Test::requireThatSmallNodesWorks() { typedef BTreeStore<MyKey, std::string, btree::NoAggregated, MyComp, BTreeDefaultTraits> TreeStore; @@ -1018,7 +1017,7 @@ Test::requireThatSmallNodesWorks(void) void -Test::requireThatApplyWorks(void) +Test::requireThatApplyWorks() { typedef BTreeStore<MyKey, std::string, btree::NoAggregated, MyComp, BTreeDefaultTraits> TreeStore; @@ -1154,11 +1153,7 @@ public: { } - int - getPathSize(void) const - { - return _pathSize; - } + int getPathSize() const { return _pathSize; } }; diff --git a/searchlib/src/tests/memoryindex/btree/frozenbtree_test.cpp b/searchlib/src/tests/memoryindex/btree/frozenbtree_test.cpp index c965cc08aa6..7a6ee5c36d3 100644 --- a/searchlib/src/tests/memoryindex/btree/frozenbtree_test.cpp +++ b/searchlib/src/tests/memoryindex/btree/frozenbtree_test.cpp @@ -49,7 +49,7 @@ private: Rand48 _randomGenerator; - void allocTree(void); + void allocTree(); void freeTree(bool verbose); void fillRandomValues(unsigned int count); void insertRandomValues(Tree &tree, NodeAllocator &allocator, const std::vector<KeyType> &values); @@ -282,7 +282,7 @@ lookupFrozenRandomValues(const Tree &tree, void -FrozenBTreeTest::sortRandomValues(void) +FrozenBTreeTest::sortRandomValues() { std::vector<KeyType>::iterator i; std::vector<KeyType>::iterator ie; diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp index 9c119b71876..c45129709bc 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("featurestore_test"); #include <vespa/vespalib/testkit/testapp.h> @@ -25,29 +24,15 @@ class Test : public vespalib::TestApp private: Schema _schema; - const Schema & - getSchema(void) const - { - return _schema; - } - - bool - assertFeatures(const DocIdAndFeatures &exp, - const DocIdAndFeatures &act); - - void - requireThatFeaturesCanBeAddedAndRetrieved(void); - - void - requireThatNextWordsAreWorking(void); - void - requireThatAddFeaturesTriggersChangeOfBuffer(void); + const Schema & getSchema() const { return _schema; } + bool assertFeatures(const DocIdAndFeatures &exp, const DocIdAndFeatures &act); + void requireThatFeaturesCanBeAddedAndRetrieved(); + void requireThatNextWordsAreWorking(); + void requireThatAddFeaturesTriggersChangeOfBuffer(); public: - Test(void); - - int - Main(void) override; + Test(); + int Main() override; }; @@ -101,7 +86,7 @@ getFeatures(uint32_t numOccs, void -Test::requireThatFeaturesCanBeAddedAndRetrieved(void) +Test::requireThatFeaturesCanBeAddedAndRetrieved() { FeatureStore fs(getSchema()); DocIdAndFeatures act; @@ -145,7 +130,7 @@ Test::requireThatFeaturesCanBeAddedAndRetrieved(void) void -Test::requireThatNextWordsAreWorking(void) +Test::requireThatNextWordsAreWorking() { FeatureStore fs(getSchema()); DocIdAndFeatures act; @@ -189,7 +174,7 @@ Test::requireThatNextWordsAreWorking(void) void -Test::requireThatAddFeaturesTriggersChangeOfBuffer(void) +Test::requireThatAddFeaturesTriggersChangeOfBuffer() { FeatureStore fs(getSchema()); size_t cnt = 1; diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp index dfe1438bcb5..f41bd36bb6c 100644 --- a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("wordstore_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp index 048851f5d94..3dbf4efc368 100644 --- a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp +++ b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp @@ -81,7 +81,7 @@ public: } virtual void - endWord(void) override + endWord() override { assert(_insideWord); assert(!_insideDoc); @@ -122,7 +122,7 @@ public: } virtual void - endDocument(void) override + endDocument() override { assert(_insideDoc); assert(!_insideElem); @@ -147,7 +147,7 @@ public: } virtual void - endElement(void) override + endElement() override { assert(_insideElem); _ss << "]"; @@ -165,7 +165,7 @@ public: } std::string - toStr(void) const + toStr() const { return _ss.str(); } @@ -1178,13 +1178,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) f._b.startDocument("doc::10"); f._b.startIndexField("iu"). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4"). + addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("81"). @@ -1202,13 +1202,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) f._b.startIndexField("iau"). startElement(1). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8"). + addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("82"). @@ -1250,13 +1250,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) f._b.startIndexField("iwu"). startElement(4). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12"). + addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("83"). @@ -1315,7 +1315,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("iu"); - PostingIterator itr(f._d.findFrozen("yahoo", fieldId), + PostingIterator itr(f._d.findFrozen("example", fieldId), featureStoreRef(f._d, fieldId), fieldId, matchData); itr.initFullRange(); @@ -1327,7 +1327,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("iau"); - PostingIterator itr(f._d.findFrozen("yahoo", fieldId), + PostingIterator itr(f._d.findFrozen("example", fieldId), featureStoreRef(f._d, fieldId), fieldId, matchData); itr.initFullRange(); @@ -1340,7 +1340,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("iwu"); - PostingIterator itr(f._d.findFrozen("yahoo", fieldId), + PostingIterator itr(f._d.findFrozen("example", fieldId), featureStoreRef(f._d, fieldId), fieldId, matchData); itr.initFullRange(); diff --git a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp b/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp index 838e876d284..ea23f0c5dbf 100644 --- a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("document_remover_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp index 89098848cc5..04de5da4b6c 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp @@ -2,7 +2,6 @@ /* -*- mode: C++; coding: utf-8; -*- */ -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/documentinverter.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp index df292ec2b1d..f7cacb4a57d 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> #include <vespa/vespalib/objects/nbostream.h> diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp index 9ed5ff16c9e..da6087d8d52 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("memoryindex_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp index c50ebc056c8..5011310f412 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp @@ -2,7 +2,6 @@ /* -*- mode: C++; coding: utf-8; -*- */ -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/memoryindex/fieldinverter.h> #include <vespa/searchlib/memoryindex/urlfieldinverter.h> @@ -31,13 +30,13 @@ makeDoc10Single(DocBuilder &b) b.startDocument("doc::10"); b.startIndexField("url"). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4"). + addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("81"). @@ -64,13 +63,13 @@ makeDoc10Array(DocBuilder &b) b.startIndexField("url"). startElement(1). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8"). + addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("82"). @@ -120,13 +119,13 @@ makeDoc10WeightedSet(DocBuilder &b) b.startIndexField("url"). startElement(4). startSubField("all"). - addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12"). + addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). endSubField(). startSubField("scheme"). addUrlTokenizedString("http"). endSubField(). startSubField("host"). - addUrlTokenizedString("www.yahoo.com"). + addUrlTokenizedString("www.example.com"). endSubField(). startSubField("port"). addUrlTokenizedString("83"). @@ -259,16 +258,16 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE)) "w=81,a=10," "w=ab,a=10," "w=com,a=10," + "w=example,a=10," "w=fluke,a=10," "w=http,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=1," "w=http,a=10," "f=2," "w=com,a=10," + "w=example,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=3," "w=81,a=10," "f=4," @@ -282,8 +281,8 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE)) "w=EnDhOsT,a=10," "w=StArThOsT,a=10," "w=com,a=10," - "w=www,a=10," - "w=yahoo,a=10", + "w=example,a=10," + "w=www,a=10", f._inserter.toStr()); } @@ -299,18 +298,18 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY)) "w=9,a=10," "w=ab,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=fluke,a=10," "w=http,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=1," "w=http,a=10," "f=2," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=3," "w=82,a=10," "f=4," @@ -325,9 +324,9 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY)) "w=EnDhOsT,a=10," "w=StArThOsT,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," - "w=www,a=10," - "w=yahoo,a=10", + "w=www,a=10", f._inserter.toStr()); } @@ -343,18 +342,18 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET)) "w=85,a=10," "w=ab,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=fluke,a=10," "w=http,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=1," "w=http,a=10," "f=2," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=3," "w=83,a=10," "w=85,a=10," @@ -370,9 +369,9 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET)) "w=EnDhOsT,a=10," "w=StArThOsT,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," - "w=www,a=10," - "w=yahoo,a=10", + "w=www,a=10", f._inserter.toStr()); } @@ -387,16 +386,16 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE "w=81,a=10," "w=ab,a=10," "w=com,a=10," + "w=example,a=10," "w=fluke,a=10," "w=http,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=1," "w=http,a=10," "f=2," "w=com,a=10," + "w=example,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=3," "w=81,a=10," "f=4," @@ -411,8 +410,8 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE "w=EnDhOsT,a=10," "w=StArThOsT,a=10," "w=com,a=10," - "w=www,a=10," - "w=yahoo,a=10", + "w=example,a=10," + "w=www,a=10", f._inserter.toStr()); } @@ -429,18 +428,18 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY)) "w=9,a=10," "w=ab,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=fluke,a=10," "w=http,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=1," "w=http,a=10," "f=2," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," "w=www,a=10," - "w=yahoo,a=10," "f=3," "w=82,a=10," "f=4," @@ -456,9 +455,9 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY)) "w=EnDhOsT,a=10," "w=StArThOsT,a=10," "w=com,a=10," + "w=example,a=10," "w=flickr,a=10," - "w=www,a=10," - "w=yahoo,a=10", + "w=www,a=10", f._inserter.toStr()); } @@ -477,18 +476,18 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks", "w=85,a=10(e=1,w=7,l=9[4])," "w=ab,a=10(e=0,w=4,l=9[6],e=1,w=7,l=9[6])," "w=com,a=10(e=0,w=4,l=9[3],e=1,w=7,l=9[3])," + "w=example,a=10(e=0,w=4,l=9[2])," "w=flickr,a=10(e=1,w=7,l=9[2])," "w=fluke,a=10(e=0,w=4,l=9[5],e=1,w=7,l=9[5])," "w=http,a=10(e=0,w=4,l=9[0],e=1,w=7,l=9[0])," "w=www,a=10(e=0,w=4,l=9[1],e=1,w=7,l=9[1])," - "w=yahoo,a=10(e=0,w=4,l=9[2])," "f=1," "w=http,a=10(e=0,w=4,l=1[0],e=1,w=7,l=1[0])," "f=2," "w=com,a=10(e=0,w=4,l=3[2],e=1,w=7,l=3[2])," + "w=example,a=10(e=0,w=4,l=3[1])," "w=flickr,a=10(e=1,w=7,l=3[1])," "w=www,a=10(e=0,w=4,l=3[0],e=1,w=7,l=3[0])," - "w=yahoo,a=10(e=0,w=4,l=3[1])," "f=3," "w=83,a=10(e=0,w=4,l=1[0])," "w=85,a=10(e=1,w=7,l=1[0])," @@ -505,9 +504,9 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks", "w=EnDhOsT,a=10(e=0,w=4,l=5[4],e=1,w=7,l=5[4])," "w=StArThOsT,a=10(e=0,w=4,l=5[0],e=1,w=7,l=5[0])," "w=com,a=10(e=0,w=4,l=5[3],e=1,w=7,l=5[3])," + "w=example,a=10(e=0,w=4,l=5[2])," "w=flickr,a=10(e=1,w=7,l=5[2])," - "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1])," - "w=yahoo,a=10(e=0,w=4,l=5[2])", + "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1])", f._inserter.toStr()); } diff --git a/searchlib/src/tests/nearsearch/nearsearch_test.cpp b/searchlib/src/tests/nearsearch/nearsearch_test.cpp index 90565ab1e1e..125c0b1278b 100644 --- a/searchlib/src/tests/nearsearch/nearsearch_test.cpp +++ b/searchlib/src/tests/nearsearch/nearsearch_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("nearsearch_test"); diff --git a/searchlib/src/tests/postinglistbm/andstress.cpp b/searchlib/src/tests/postinglistbm/andstress.cpp index 69aa463b46d..dbe7ee1a191 100644 --- a/searchlib/src/tests/postinglistbm/andstress.cpp +++ b/searchlib/src/tests/postinglistbm/andstress.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP(".andstress"); #include "andstress.h" @@ -71,45 +70,19 @@ public: uint32_t stride, bool unpack); - ~AndStressMaster(void); - - void - run(void); - - void - makePostingsHelper(FPFactory *postingFactory, - const std::string &postingFormat, - bool validate, bool verbose); - - void - dropPostings(void); - - void - dropTasks(void); - - void - resetTasks(void); // Prepare for rerun - - void - setupTasks(unsigned int numTasks); - - Task * - getTask(void); - - unsigned int - getNumDocs(void) const - { - return _numDocs; - } - - bool - getUnpack(void) const - { - return _unpack; - } - - double - runWorkers(const std::string &postingFormat); + ~AndStressMaster(); + void run(); + void makePostingsHelper(FPFactory *postingFactory, + const std::string &postingFormat, + bool validate, bool verbose); + void dropPostings(); + void dropTasks(); + void resetTasks(); // Prepare for rerun + void setupTasks(unsigned int numTasks); + Task *getTask(); + unsigned int getNumDocs() const { return _numDocs; } + bool getUnpack() const { return _unpack; } + double runWorkers(const std::string &postingFormat); }; @@ -125,11 +98,8 @@ private: unsigned int _id; public: AndStressWorker(AndStressMaster &master, unsigned int id); - - ~AndStressWorker(void); - - virtual void - Run(FastOS_ThreadInterface *thisThread, void *arg) override; + ~AndStressWorker(); + virtual void Run(FastOS_ThreadInterface *thisThread, void *arg) override; }; @@ -184,7 +154,7 @@ clearPtrVector(std::vector<C> &v) } -AndStressMaster::~AndStressMaster(void) +AndStressMaster::~AndStressMaster() { LOG(info, "AndStressMaster::~AndStressMaster"); @@ -197,7 +167,7 @@ AndStressMaster::~AndStressMaster(void) void -AndStressMaster::dropPostings(void) +AndStressMaster::dropPostings() { for (unsigned int i = 0; i < _postings.size(); ++i) _postings[i].clear(); @@ -206,7 +176,7 @@ AndStressMaster::dropPostings(void) void -AndStressMaster::dropTasks(void) +AndStressMaster::dropTasks() { _tasks.clear(); _taskIdx = 0; @@ -214,7 +184,7 @@ AndStressMaster::dropTasks(void) void -AndStressMaster::resetTasks(void) +AndStressMaster::resetTasks() { _taskIdx = 0; } @@ -302,7 +272,7 @@ AndStressMaster::setupTasks(unsigned int numTasks) AndStressMaster::Task * -AndStressMaster::getTask(void) +AndStressMaster::getTask() { Task *result = NULL; _taskCond.Lock(); @@ -319,7 +289,7 @@ AndStressMaster::getTask(void) } void -AndStressMaster::run(void) +AndStressMaster::run() { LOG(info, "AndStressMaster::run"); @@ -381,7 +351,7 @@ AndStressWorker::AndStressWorker(AndStressMaster &master, unsigned int id) LOG(debug, "AndStressWorker::AndStressWorker, id=%u", id); } -AndStressWorker::~AndStressWorker(void) +AndStressWorker::~AndStressWorker() { LOG(debug, "AndStressWorker::~AndStressWorker, id=%u", _id); } @@ -500,13 +470,13 @@ AndStressWorker::Run(FastOS_ThreadInterface *thisThread, void *arg) } -AndStress::AndStress(void) +AndStress::AndStress() { LOG(debug, "Andstress::AndStress"); } -AndStress::~AndStress(void) +AndStress::~AndStress() { LOG(debug, "Andstress::~AndStress"); } diff --git a/searchlib/src/tests/postinglistbm/andstress.h b/searchlib/src/tests/postinglistbm/andstress.h index 458866b09d5..dd9c4276c2c 100644 --- a/searchlib/src/tests/postinglistbm/andstress.h +++ b/searchlib/src/tests/postinglistbm/andstress.h @@ -22,9 +22,9 @@ namespace postinglistbm class AndStress { public: - AndStress(void); + AndStress(); - ~AndStress(void); + ~AndStress(); void run(search::Rand48 &rnd, diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index ff09a99ba99..02007e1ae64 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -2,7 +2,6 @@ // Copyright (C) 2002-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/util/rand48.h> @@ -12,6 +11,7 @@ #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> #include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/fastos/app.h> using search::ResultSet; using search::fef::TermFieldMatchData; @@ -27,8 +27,7 @@ using namespace search::fakedata; void FastS_block_usr2() {} -namespace postinglistbm -{ +namespace postinglistbm { class PostingListBM : public FastOS_Application { @@ -47,24 +46,20 @@ public: search::Rand48 _rnd; private: - void Usage(void); - - void - badPostingType(const std::string &postingType); - - void - testFake(const std::string &postingType, - const Schema &schema, - const FakeWord &fw); + void Usage(); + void badPostingType(const std::string &postingType); + void testFake(const std::string &postingType, + const Schema &schema, + const FakeWord &fw); public: - PostingListBM(void); - ~PostingListBM(void); - int Main(void) override; + PostingListBM(); + ~PostingListBM(); + int Main() override; }; void -PostingListBM::Usage(void) +PostingListBM::Usage() { printf("postinglistbm " "[-C <skipCommonPairsRate>] " @@ -102,7 +97,7 @@ PostingListBM::badPostingType(const std::string &postingType) } -PostingListBM::PostingListBM(void) +PostingListBM::PostingListBM() : _verbose(false), _numDocs(10000000), _commonDocFreq(50000), @@ -118,7 +113,7 @@ PostingListBM::PostingListBM(void) } -PostingListBM::~PostingListBM(void) +PostingListBM::~PostingListBM() { } @@ -323,7 +318,7 @@ testFakePair(const std::string &postingType, int -PostingListBM::Main(void) +PostingListBM::Main() { int argi; char c; diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index 72b93dc4d08..ae449dade19 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("document_features_store_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/document_features_store.h> #include <vespa/searchlib/predicate/predicate_index.h> diff --git a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp index 26d4050f440..672d6dcaccc 100644 --- a/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_bounds_posting_list_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_bounds_posting_list_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_bounds_posting_list.h> diff --git a/searchlib/src/tests/predicate/predicate_index_test.cpp b/searchlib/src/tests/predicate/predicate_index_test.cpp index cf5c1d6bfb5..5197edf3fee 100644 --- a/searchlib/src/tests/predicate/predicate_index_test.cpp +++ b/searchlib/src/tests/predicate/predicate_index_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_index_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> diff --git a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp index 831e24026ec..f00cb339803 100644 --- a/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_posting_list_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_interval_posting_list_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_interval_posting_list.h> diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp index bfe6340e222..f3957a70c51 100644 --- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_interval_store_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_interval_store.h> diff --git a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp index 47c29184dcf..09022e7965f 100644 --- a/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp +++ b/searchlib/src/tests/predicate/predicate_range_term_expander_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_range_term_expander_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_range_term_expander.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp index a51f3d678d5..a0fdb622020 100644 --- a/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp +++ b/searchlib/src/tests/predicate/predicate_ref_cache_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_ref_cache_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_ref_cache.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp index f455abced3f..d2e57cd4655 100644 --- a/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_analyzer_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("PredicateTreeAnalyzer_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_slime_builder.h> diff --git a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp index 92271cd1c20..dac6b483120 100644 --- a/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp +++ b/searchlib/src/tests/predicate/predicate_tree_annotator_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("PredicateTreeAnnotator_test"); -#include <vespa/fastos/fastos.h> #include <vespa/document/predicate/predicate.h> #include <vespa/document/predicate/predicate_slime_builder.h> diff --git a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp index 90e34c804e7..f656b97cb0e 100644 --- a/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zero_constraint_posting_list_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_zero_constraint_posting_list_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_zero_constraint_posting_list.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp index dfa75d48019..01a6d88ac47 100644 --- a/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp +++ b/searchlib/src/tests/predicate/predicate_zstar_compressed_posting_list_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_zstar_compressed_posting_list_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_zstar_compressed_posting_list.h> diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index 933f21ebf3c..0279ed25a81 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for simple_index. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/simple_index.hpp> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/attribute/predicate_attribute.h> diff --git a/searchlib/src/tests/predicate/tree_crumbs_test.cpp b/searchlib/src/tests/predicate/tree_crumbs_test.cpp index 2f38bb74507..47f0ca13cbc 100644 --- a/searchlib/src/tests/predicate/tree_crumbs_test.cpp +++ b/searchlib/src/tests/predicate/tree_crumbs_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("TreeCrumbs_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/predicate/tree_crumbs.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/prettyfloat/prettyfloat.cpp b/searchlib/src/tests/prettyfloat/prettyfloat.cpp index 1ed9b7e1767..0398a463c54 100644 --- a/searchlib/src/tests/prettyfloat/prettyfloat.cpp +++ b/searchlib/src/tests/prettyfloat/prettyfloat.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("prettyfloat_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 75abcc810b2..a3e5c0efa12 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for customtypevisitor. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("customtypevisitor_test"); diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index c54b9580ea8..345d374865e 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for query_visitor. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("query_visitor_test"); diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 818324ab3b8..0edf24342da 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querybuilder. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("querybuilder_test"); diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index 269947b7059..c7a29e299cd 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for stackdumpquerycreator. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("stackdumpquerycreator_test"); diff --git a/searchlib/src/tests/query/templatetermvisitor_test.cpp b/searchlib/src/tests/query/templatetermvisitor_test.cpp index 5415884aa31..1e5131e2491 100644 --- a/searchlib/src/tests/query/templatetermvisitor_test.cpp +++ b/searchlib/src/tests/query/templatetermvisitor_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for templatetermvisitor. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("templatetermvisitor_test"); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index d17ab183c0c..4644aca283c 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("blueprint_test"); #include <vespa/vespalib/testkit/testapp.h> @@ -34,7 +33,7 @@ public: } virtual bool inheritStrict(size_t i) const override { - (void)i; + (void) i; return true; } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index ae856f7d4a0..1c49903f00b 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("blueprint_test"); diff --git a/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp index 513100dc4d3..edec38db79c 100644 --- a/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/leaf_blueprints_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("blueprint_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp index 48379789841..0d3ed403abd 100644 --- a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp +++ b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("equiv_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp index 0e9fcff0073..7b9e5234e0a 100644 --- a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp +++ b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("getweight_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp index 0a6586a31d2..b3aacd26706 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("multibitvectoriterator_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index 3b609849141..01851aaa9ef 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_blueprint_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/predicate_attribute.h> diff --git a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp index 5954d51ec9b..2b8c01ded2f 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_search_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("predicate_search_test"); -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 470b309db0f..ad2fe622677 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("simple_phrase_test"); diff --git a/searchlib/src/tests/queryeval/weak_and/parallel_weak_and_bench.cpp b/searchlib/src/tests/queryeval/weak_and/parallel_weak_and_bench.cpp index 8e5b7fc7b85..99f15ab3a18 100644 --- a/searchlib/src/tests/queryeval/weak_and/parallel_weak_and_bench.cpp +++ b/searchlib/src/tests/queryeval/weak_and/parallel_weak_and_bench.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wand_bench_setup.hpp" TEST_FF("benchmark", VespaParallelWandFactory(1000), WandSetup(f1, 10, 10000000)) { f2.benchmark(); } diff --git a/searchlib/src/tests/queryeval/weak_and/weak_and_bench.cpp b/searchlib/src/tests/queryeval/weak_and/weak_and_bench.cpp index 1eba66a524f..af6dbb3eee9 100644 --- a/searchlib/src/tests/queryeval/weak_and/weak_and_bench.cpp +++ b/searchlib/src/tests/queryeval/weak_and/weak_and_bench.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wand_bench_setup.hpp" TEST_FF("benchmark", VespaWandFactory(1000), WandSetup(f1, 10, 10000000)) { f2.benchmark(); } diff --git a/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp b/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp index 8f60b6dd8c7..30deb376e05 100644 --- a/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp +++ b/searchlib/src/tests/queryeval/weak_and/weak_and_test_expensive.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wand_bench_setup.hpp" using namespace rise; diff --git a/searchlib/src/tests/queryeval/weak_and_heap/weak_and_heap_test.cpp b/searchlib/src/tests/queryeval/weak_and_heap/weak_and_heap_test.cpp index ee44abf2b27..dd09c30921f 100644 --- a/searchlib/src/tests/queryeval/weak_and_heap/weak_and_heap_test.cpp +++ b/searchlib/src/tests/queryeval/weak_and_heap/weak_and_heap_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/queryeval/wand/weak_and_heap.h> diff --git a/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp b/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp index 55e299462b9..9547f553e7b 100644 --- a/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp +++ b/searchlib/src/tests/queryeval/weak_and_scorers/weak_and_scorers_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/queryeval/searchiterator.h> diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp index 1e49cd4aae6..c0d013b6542 100644 --- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp +++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/indexenvironment.h> diff --git a/searchlib/src/tests/sort/uca.cpp b/searchlib/src/tests/sort/uca.cpp index c4802f5d3ce..d8a15242b40 100644 --- a/searchlib/src/tests/sort/uca.cpp +++ b/searchlib/src/tests/sort/uca.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/sort.h> diff --git a/searchlib/src/tests/sortresults/sorttest.cpp b/searchlib/src/tests/sortresults/sorttest.cpp index 4c3326f4cbc..e2921bb1ece 100644 --- a/searchlib/src/tests/sortresults/sorttest.cpp +++ b/searchlib/src/tests/sortresults/sorttest.cpp @@ -3,7 +3,6 @@ // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/sortresults.h> diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index 326b2c0b280..190f8dda2fe 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -2,10 +2,10 @@ // Copyright (C) 2001-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> +#include "stackdumpiteratortest.h" #include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> -#include "stackdumpiteratortest.h" +#include <vespa/fastos/time.h> #define NUMTESTS 5 diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h index 121653a89d1..0d1ff63dd56 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h @@ -5,6 +5,9 @@ #pragma once #include <vespa/searchlib/parsequery/stackdumpiterator.h> +#include <vespa/fastos/app.h> + +namespace search { class SimpleQueryStack; } class StackDumpIteratorTest : public FastOS_Application { @@ -13,4 +16,3 @@ class StackDumpIteratorTest : public FastOS_Application bool ShowResult(int testNo, search::SimpleQueryStackDumpIterator &actual, search::SimpleQueryStack &correct, unsigned int expected); bool RunTest(int i, bool verify); }; - diff --git a/searchlib/src/tests/stringenum/stringenum_test.cpp b/searchlib/src/tests/stringenum/stringenum_test.cpp index 264fd964a04..3d08bc34ec6 100644 --- a/searchlib/src/tests/stringenum/stringenum_test.cpp +++ b/searchlib/src/tests/stringenum/stringenum_test.cpp @@ -3,7 +3,6 @@ // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("stringenum"); #include <vespa/fastlib/io/bufferedfile.h> @@ -19,8 +18,7 @@ class MyApp : public vespalib::TestApp public: void CheckLookup( search::util::StringEnum *strEnum, const char *str, int value); int Main() override; - - MyApp(void) {} + MyApp() {} }; diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 628958a921a..b39cd73b357 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("dense_tensor_store_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/searchlib/src/tests/transactionlog/.gitignore b/searchlib/src/tests/transactionlog/.gitignore index a7bdcf0397d..9f36bea89dc 100644 --- a/searchlib/src/tests/transactionlog/.gitignore +++ b/searchlib/src/tests/transactionlog/.gitignore @@ -2,6 +2,5 @@ Makefile translogserver_test translogclient_test - searchlib_translogclient_test_app searchlib_translogserver_test_app diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index bd0053827d2..ee9ecf09efb 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 3719c5da1c7..b4285da447b 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -9,6 +9,7 @@ #include <vespa/fastos/app.h> #include <iostream> #include <stdexcept> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("translogstress"); diff --git a/searchlib/src/tests/true/true.cpp b/searchlib/src/tests/true/true.cpp index d052f3ca4e4..ec88a83d2c2 100644 --- a/searchlib/src/tests/true/true.cpp +++ b/searchlib/src/tests/true/true.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("true_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/url/testurl.cpp b/searchlib/src/tests/url/testurl.cpp index bfe6c79de07..5a168032986 100644 --- a/searchlib/src/tests/url/testurl.cpp +++ b/searchlib/src/tests/url/testurl.cpp @@ -2,9 +2,10 @@ // Copyright (C) 2000-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/util/url.h> - +#include <cstdio> +#include <cstring> +#include <cassert> // needed to resolve external symbol from httpd.h on AIX void FastS_block_usr2() {} diff --git a/searchlib/src/tests/util/bufferwriter/bm.cpp b/searchlib/src/tests/util/bufferwriter/bm.cpp index 228ce4adcdc..73ee7906af9 100644 --- a/searchlib/src/tests/util/bufferwriter/bm.cpp +++ b/searchlib/src/tests/util/bufferwriter/bm.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("bufferwriter_bm"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/util/bufferwriter/work.cpp b/searchlib/src/tests/util/bufferwriter/work.cpp index 9835b28970e..8521d451913 100644 --- a/searchlib/src/tests/util/bufferwriter/work.cpp +++ b/searchlib/src/tests/util/bufferwriter/work.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "work.h" #include <vespa/searchlib/util/bufferwriter.h> diff --git a/searchlib/src/tests/util/bufferwriter/work.h b/searchlib/src/tests/util/bufferwriter/work.h index 49a1cb8017d..ef58d2e523a 100644 --- a/searchlib/src/tests/util/bufferwriter/work.h +++ b/searchlib/src/tests/util/bufferwriter/work.h @@ -1,5 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> + namespace search { class BufferWriter; @@ -16,4 +20,3 @@ template <class T> void workFunctor2(const std::vector<T> &v, BufferWriter &writer); } // namespace search - diff --git a/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp b/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp index bcfaa5f8ac8..84d59010ceb 100644 --- a/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp +++ b/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp @@ -1,42 +1,40 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("ioerrorhandler_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/util/statefile.h> #include <vespa/searchlib/util/ioerrorhandler.h> +#include <vespa/searchlib/test/statefile.h> +#include <vespa/searchlib/test/statestring.h> +#include <vespa/fastos/file.h> #include <atomic> #include <iostream> #include <fstream> #include <string> #include <setjmp.h> #include <dlfcn.h> -#include <vespa/searchlib/test/statefile.h> -#include <vespa/searchlib/test/statestring.h> -extern "C" -{ +#include <vespa/log/log.h> +LOG_SETUP("ioerrorhandler_test"); + +extern "C" { ssize_t read(int fd, void *buf, size_t count); ssize_t write(int fd, const void *buf, size_t count); ssize_t pread(int fd, void *buf, size_t count, off_t offset); ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); - } using ReadFunc = ssize_t (*)(int fd, void *buf, size_t count); using WriteFunc = ssize_t (*)(int fd, const void *buf, size_t count); using PreadFunc = ssize_t (*)(int fd, void *buf, size_t count, off_t offset); -using PwriteFunc = ssize_t (*)(int fd, const void *buf, size_t count, - off_t offset); +using PwriteFunc = ssize_t (*)(int fd, const void *buf, size_t count, off_t offset); using namespace search::test::statefile; using namespace search::test::statestring; -namespace -{ +namespace { ReadFunc libc_read; WriteFunc libc_write; diff --git a/searchlib/src/tests/util/rawbuf_test.cpp b/searchlib/src/tests/util/rawbuf_test.cpp index e9dc139bda5..6423e736998 100644 --- a/searchlib/src/tests/util/rawbuf_test.cpp +++ b/searchlib/src/tests/util/rawbuf_test.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for rawbuf. -#include <vespa/log/log.h> -LOG_SETUP("rawbuf_test"); -#include <vespa/fastos/fastos.h> - #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP("rawbuf_test"); using vespalib::string; using namespace search; diff --git a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp index 3aa3b89b19a..3b1e068f637 100644 --- a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp +++ b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("searchable_stats_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp b/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp index af657420575..da214d0f67c 100644 --- a/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp +++ b/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("sigbushandler_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/util/statebuf/statebuf_test.cpp b/searchlib/src/tests/util/statebuf/statebuf_test.cpp index 952412b9eb6..d1b0bee75d9 100644 --- a/searchlib/src/tests/util/statebuf/statebuf_test.cpp +++ b/searchlib/src/tests/util/statebuf/statebuf_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("statebuf_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/util/statefile/statefile_test.cpp b/searchlib/src/tests/util/statefile/statefile_test.cpp index b932ec03bb2..32f8e061048 100644 --- a/searchlib/src/tests/util/statefile/statefile_test.cpp +++ b/searchlib/src/tests/util/statefile/statefile_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("statefile_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/fileheaderinspect/.gitignore b/searchlib/src/tests/vespa-fileheader-inspect/.gitignore index 812991d07b5..4ff47722dca 100644 --- a/searchlib/src/tests/fileheaderinspect/.gitignore +++ b/searchlib/src/tests/vespa-fileheader-inspect/.gitignore @@ -3,4 +3,4 @@ Makefile fileheader.dat fileheaderinspect_test out -searchlib_fileheaderinspect_test_app +searchlib_vespa-fileheader-inspect_test_app diff --git a/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt new file mode 100644 index 00000000000..14bf5a82e3f --- /dev/null +++ b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_vespa-fileheader-inspect_test_app TEST + SOURCES + vespa-fileheader-inspect_test.cpp + DEPENDS + searchlib + AFTER + searchlib_vespa-fileheader-inspect_app +) +vespa_add_test(NAME searchlib_vespa-fileheader-inspect_test_app COMMAND searchlib_vespa-fileheader-inspect_test_app) diff --git a/searchlib/src/tests/fileheaderinspect/DESC b/searchlib/src/tests/vespa-fileheader-inspect/DESC index ee57a2fdde3..ee57a2fdde3 100644 --- a/searchlib/src/tests/fileheaderinspect/DESC +++ b/searchlib/src/tests/vespa-fileheader-inspect/DESC diff --git a/searchlib/src/tests/fileheaderinspect/FILES b/searchlib/src/tests/vespa-fileheader-inspect/FILES index 7c32fb811d5..7c32fb811d5 100644 --- a/searchlib/src/tests/fileheaderinspect/FILES +++ b/searchlib/src/tests/vespa-fileheader-inspect/FILES diff --git a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp index 68bae5e57d8..b54b06e4d93 100644 --- a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp +++ b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp @@ -34,14 +34,14 @@ vespalib::string readFile(const vespalib::string &fileName) { } TEST("testError") { - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect notfound.dat") != 0); + EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect notfound.dat") != 0); } TEST("testEscape") { FileHeader header; header.putTag(FileHeader::Tag("fanart", "\fa\na\r\t")); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -q fileheader.dat > out") == 0); EXPECT_EQUAL("fanart;string;\\fa\\na\\r\\t\n", readFile("out")); } @@ -49,7 +49,7 @@ TEST("testDelimiter") { FileHeader header; header.putTag(FileHeader::Tag("string", "string")); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -d i -q fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -d i -q fileheader.dat > out") == 0); EXPECT_EQUAL("str\\ingistr\\ingistr\\ing\n", readFile("out")); } @@ -57,7 +57,7 @@ TEST("testQuiet") { FileHeader header; FileHeaderTk::addVersionTags(header); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -q fileheader.dat > out") == 0); vespalib::string str = readFile("out"); EXPECT_TRUE(!str.empty()); for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) { @@ -75,7 +75,7 @@ TEST("testVerbose") { FileHeader header; FileHeaderTk::addVersionTags(header); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect fileheader.dat > out") == 0); vespalib::string str = readFile("out"); EXPECT_TRUE(!str.empty()); for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) { diff --git a/searchlib/src/vespa/searchlib/aggregation/fs4hit.cpp b/searchlib/src/vespa/searchlib/aggregation/fs4hit.cpp index f63f44d3a12..aac21a0ae66 100644 --- a/searchlib/src/vespa/searchlib/aggregation/fs4hit.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/fs4hit.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fs4hit.h" #include <vespa/vespalib/objects/visit.h> diff --git a/searchlib/src/vespa/searchlib/aggregation/hit.cpp b/searchlib/src/vespa/searchlib/aggregation/hit.cpp index 9bb7cf95757..501249cf867 100644 --- a/searchlib/src/vespa/searchlib/aggregation/hit.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/hit.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "hit.h" #include <vespa/vespalib/objects/visit.h> diff --git a/searchlib/src/vespa/searchlib/aggregation/rawrank.cpp b/searchlib/src/vespa/searchlib/aggregation/rawrank.cpp index b570c7795c0..7674cde1de7 100644 --- a/searchlib/src/vespa/searchlib/aggregation/rawrank.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/rawrank.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "rawrank.h" #include <vespa/vespalib/util/sort.h> #include <algorithm> diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index e06514e3fc0..0e7dee7d668 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -67,6 +67,7 @@ vespa_add_library(searchlib_attribute OBJECT multistringpostattribute.cpp multivalueattribute.cpp multivalueattributesaver.cpp + multivalueattributesaverutils.cpp not_implemented_attribute.cpp numericbase.cpp postingchange.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute.cpp b/searchlib/src/vespa/searchlib/attribute/attribute.cpp index 649ac1c9786..e0a47db521f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attribute.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.attribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h index e761122d28f..aae875c7e19 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h @@ -59,10 +59,10 @@ public: bool hasWeightedSetType() const; uint32_t getNumDocs() const { return _numDocs; } size_t getFixedWidth() const { return _fixedWidth; } - uint64_t getUniqueValueCount(void) const { return _uniqueValueCount; } - uint64_t getTotalValueCount(void) const { return _totalValueCount; } - bool getEnumerated(void) const { return _enumerated; } - uint64_t getCreateSerialNum(void) const { return _createSerialNum; } + uint64_t getUniqueValueCount() const { return _uniqueValueCount; } + uint64_t getTotalValueCount() const { return _totalValueCount; } + bool getEnumerated() const { return _enumerated; } + uint64_t getCreateSerialNum() const { return _createSerialNum; } uint32_t getVersion() const { return _version; } const PersistentPredicateParams &getPredicateParams() const { return _predicateParams; } bool getPredicateParamsSet() const { return _predicateParamsSet; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp b/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp index f7d5adbe049..e80fd2ab8c0 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "attributefactory.h" + #include <vespa/log/log.h> LOG_SETUP(".searchlib.attributefactory"); -#include "attributefactory.h" - namespace search { using attribute::CollectionType; diff --git a/searchlib/src/vespa/searchlib/attribute/attributefactory.h b/searchlib/src/vespa/searchlib/attribute/attributefactory.h index d2a1f567620..d7a76db8809 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefactory.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefactory.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/attribute/attributevector.h> +#include "attributevector.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/attributefile.cpp b/searchlib/src/vespa/searchlib/attribute/attributefile.cpp index c4384459f9d..8394fc36fb6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefile.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefile.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributefile.h" -#include <stdexcept> +#include <vespa/searchlib/util/filesizecalculator.h> #include <vespa/vespalib/util/error.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/util/filesizecalculator.h> +#include <stdexcept> #include <vespa/log/log.h> LOG_SETUP(".attributefile"); @@ -15,13 +14,11 @@ using vespalib::IllegalStateException; using search::common::FileHeaderContext; using vespalib::getLastErrorString; -namespace search -{ +namespace search { using attribute::BasicType; -namespace -{ +namespace { void updateHeader(const vespalib::string &name) @@ -258,7 +255,7 @@ AttributeFile::AttributeFile(const vespalib::string &fileName, } -AttributeFile::~AttributeFile(void) +AttributeFile::~AttributeFile() { Close(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefile.h b/searchlib/src/vespa/searchlib/attribute/attributefile.h index 055424d779e..76b39c43139 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefile.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefile.h @@ -63,7 +63,7 @@ protected: typedef attribute::Config Config; public: AttributeFile(const vespalib::string & fileName, const Config & config); - ~AttributeFile(void); + ~AttributeFile(); std::unique_ptr<Record> getRecord(); bool read(Record & record); @@ -72,7 +72,7 @@ public: protected: void OpenReadOnly(); void OpenWriteOnly(const search::common::FileHeaderContext &fileHeaderContext, uint32_t docIdLimit); - void Close(void); + void Close(); bool seekIdxPos(size_t idxPos); private: uint32_t _currIdx; diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp index 15eceae889d..561993c7bce 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributefilebufferwriter.h" -namespace search -{ +namespace search { AttributeFileBufferWriter:: AttributeFileBufferWriter(IAttributeFileWriter &fileWriter) diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp index ce77c3b8722..291f265c484 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp @@ -1,20 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.attributefilesavetarget"); - #include "attributefilesavetarget.h" #include "attributevector.h" +#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/util/error.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.attribute.attributefilesavetarget"); + using vespalib::getLastErrorString; -namespace search -{ +namespace search { using common::FileHeaderContext; diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp index 0fac4d2461e..35986da9257 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp @@ -138,7 +138,7 @@ doUnpack(uint32_t docId) template <> void AttributePostingListIteratorT<InnerAttributePostingListIterator>:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); @@ -150,7 +150,7 @@ setupPostingInfo(void) template <> void AttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { const btree::MinMaxAggregated &a(_iterator.getAggregated()); @@ -163,7 +163,7 @@ setupPostingInfo(void) template <> void AttributePostingListIteratorT<DocIdMinMaxIterator<AttributePosting> >:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); @@ -175,7 +175,7 @@ setupPostingInfo(void) template <> void AttributePostingListIteratorT<DocIdMinMaxIterator<AttributeWeightPosting> >:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { const btree::MinMaxAggregated a(_iterator.getAggregated()); @@ -187,7 +187,7 @@ setupPostingInfo(void) template <> void FilterAttributePostingListIteratorT<InnerAttributePostingListIterator>:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); @@ -199,7 +199,7 @@ setupPostingInfo(void) template <> void FilterAttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); @@ -211,7 +211,7 @@ setupPostingInfo(void) template <> void FilterAttributePostingListIteratorT<DocIdMinMaxIterator<AttributePosting> >:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); @@ -223,7 +223,7 @@ setupPostingInfo(void) template <> void FilterAttributePostingListIteratorT<DocIdMinMaxIterator<AttributeWeightPosting> >:: -setupPostingInfo(void) +setupPostingInfo() { if (_iterator.valid()) { _postingInfo = MinMaxPostingInfo(1, 1); diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index 31ac8f67afe..c0c96b24a9a 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -108,7 +108,7 @@ AttributeManager::setBaseDir(const string & base) } -AttributeManager::~AttributeManager(void) +AttributeManager::~AttributeManager() { _attributes.clear(); LOG(debug, diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h index ebdd738c968..ebba3709fbb 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h @@ -51,7 +51,7 @@ public: const string & getBaseDir() const { return _baseDir; } void setSnapshot(const Snapshot &snap) { _snapShot = snap; } void setBaseDir(const string & base); - bool hasReaders(void) const; + bool hasReaders() const; uint64_t getMemoryFootprint() const; protected: typedef vespalib::hash_map<string, VectorHolder> AttributeMap; diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp index 534a7d6ff2f..833171328a6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributememoryfilebufferwriter.h" -namespace search -{ - +namespace search { AttributeMemoryFileBufferWriter:: AttributeMemoryFileBufferWriter(IAttributeFileWriter &memoryFileWriter) diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp index a9b72350c7e..b536cdae0b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributememoryfilewriter.h" #include "attributememoryfilebufferwriter.h" -namespace search -{ +namespace search { -namespace -{ +namespace { const uint32_t MIN_ALIGNMENT = 4096; diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp index 41166c8b76b..e93e2ee34db 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.attributememorysavetarget"); - #include "attributememorysavetarget.h" #include "attributefilesavetarget.h" #include "attributevector.h" -namespace search -{ +namespace search { using search::common::FileHeaderContext; diff --git a/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp index f00a14127aa..f03fecce67c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "attributesaver.h" #include "iattributesavetarget.h" - using vespalib::GenerationHandler; -namespace search -{ +namespace search { AttributeSaver::AttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header) diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 6ba598f1f3e..6abf0ac5f27 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -198,7 +198,7 @@ size_t AttributeVector::getFixedWidth() const { return _config.basicType().fixed bool AttributeVector::hasEnum() const { return _hasEnum; } bool AttributeVector::hasEnum2Value() const { return false; } uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount; } -uint32_t AttributeVector::getNumDocs(void) const { return _status.getNumDocs(); } +uint32_t AttributeVector::getNumDocs() const { return _status.getNumDocs(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) @@ -511,7 +511,7 @@ AttributeVector::hasLoadData() const { bool -AttributeVector::isEnumeratedSaveFormat(void) const +AttributeVector::isEnumeratedSaveFormat() const { vespalib::string datName(vespalib::make_string("%s.dat", getBaseFileName().c_str())); Fast_BufferedFile datFile; @@ -663,7 +663,7 @@ AttributeVector::performCompactionWarning() void -AttributeVector::addReservedDoc(void) +AttributeVector::addReservedDoc() { uint32_t docId = 42; addDoc(docId); // Reserved @@ -701,13 +701,13 @@ bool AttributeVector::hasPostings() { return getIPostingListAttributeBase() != n uint64_t AttributeVector::getUniqueValueCount() const { return getTotalValueCount(); } uint64_t AttributeVector::getTotalValueCount() const { return getNumDocs(); } void AttributeVector::setCreateSerialNum(uint64_t createSerialNum) { _createSerialNum = createSerialNum; } -uint64_t AttributeVector::getCreateSerialNum(void) const { return _createSerialNum; } +uint64_t AttributeVector::getCreateSerialNum() const { return _createSerialNum; } uint32_t AttributeVector::getVersion() const { return 0; } void AttributeVector::compactLidSpace(uint32_t wantedLidLimit) { commit(); - assert(_uncommittedDocIdLimit <= wantedLidLimit); + assert(_committedDocIdLimit >= wantedLidLimit); if (wantedLidLimit < _committedDocIdLimit) { clearDocs(wantedLidLimit, _committedDocIdLimit); } @@ -719,17 +719,20 @@ AttributeVector::compactLidSpace(uint32_t wantedLidLimit) { bool -AttributeVector::canShrinkLidSpace(void) const { +AttributeVector::canShrinkLidSpace() const { return wantShrinkLidSpace() && _compactLidSpaceGeneration < getFirstUsedGeneration(); } void -AttributeVector::shrinkLidSpace(void) +AttributeVector::shrinkLidSpace() { commit(); - assert(canShrinkLidSpace()); + removeAllOldGenerations(); + if (!canShrinkLidSpace()) { + return; + } uint32_t committedDocIdLimit = _committedDocIdLimit; clearDocs(committedDocIdLimit, getNumDocs()); commit(); @@ -743,7 +746,7 @@ AttributeVector::shrinkLidSpace(void) updateStat(true); } -void AttributeVector::onShrinkLidSpace(void) {} +void AttributeVector::onShrinkLidSpace() {} void AttributeVector::clearDocs(DocId lidLow, DocId lidLimit) @@ -848,6 +851,33 @@ AttributeVector::getEstimatedSaveByteSize() const return datFileSize + weightFileSize + idxFileSize + udatFileSize; } +size_t +AttributeVector::getEstimatedShrinkLidSpaceGain() const +{ + size_t canFree = 0; + if (canShrinkLidSpace()) { + uint32_t committedDocIdLimit = getCommittedDocIdLimit(); + uint32_t numDocs = getNumDocs(); + const attribute::Config &cfg = getConfig(); + if (committedDocIdLimit < numDocs) { + size_t elemSize = 4; + if (!cfg.collectionType().isMultiValue() && !cfg.fastSearch()) { + BasicType::Type basicType(getBasicType()); + switch (basicType) { + case BasicType::Type::PREDICATE: + case BasicType::Type::TENSOR: + case BasicType::Type::REFERENCE: + break; + default: + elemSize = cfg.basicType().fixedSize(); + } + } + canFree = elemSize * (numDocs - committedDocIdLimit); + } + } + return canFree; +} + MemoryUsage AttributeVector::getChangeVectorMemoryUsage() const { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 94c17662a4c..523cae645e5 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -454,12 +454,12 @@ public: /** Returns whether this attribute has load data files on disk **/ bool hasLoadData() const; - bool isEnumeratedSaveFormat(void) const; + bool isEnumeratedSaveFormat() const; bool load(); void commit(bool forceStatUpdate = false); void commit(uint64_t firstSyncToken, uint64_t lastSyncToken); void setCreateSerialNum(uint64_t createSerialNum); - uint64_t getCreateSerialNum(void) const; + uint64_t getCreateSerialNum() const; virtual uint32_t getVersion() const; ////// Interface to access single documents. @@ -665,10 +665,11 @@ public: virtual uint64_t getTotalValueCount() const; virtual void compactLidSpace(uint32_t wantedLidLimit) override; virtual void clearDocs(DocId lidLow, DocId lidLimit); - bool wantShrinkLidSpace(void) const { return _committedDocIdLimit < getNumDocs(); } + bool wantShrinkLidSpace() const { return _committedDocIdLimit < getNumDocs(); } virtual bool canShrinkLidSpace() const override; virtual void shrinkLidSpace() override; virtual void onShrinkLidSpace(); + virtual size_t getEstimatedShrinkLidSpaceGain() const override; void setInterlock(const std::shared_ptr<attribute::Interlock> &interlock); diff --git a/searchlib/src/vespa/searchlib/attribute/componentguard.h b/searchlib/src/vespa/searchlib/attribute/componentguard.h index 158c3e4264a..13c7934ffb5 100644 --- a/searchlib/src/vespa/searchlib/attribute/componentguard.h +++ b/searchlib/src/vespa/searchlib/attribute/componentguard.h @@ -30,7 +30,7 @@ public: ComponentGuard(const Component & component); const T * get() const { return _component.get(); } - const Component & getSP(void) const { return _component; } + const Component & getSP() const { return _component; } const T * operator -> () const { return _component.get(); } const T & operator * () const { return *_component.get(); } T * get() { return _component.get(); } diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 9e3c6156dbf..63c964c3bf9 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.configconverter"); #include "configconverter.h" using namespace vespa::config::search; diff --git a/searchlib/src/vespa/searchlib/attribute/defines.cpp b/searchlib/src/vespa/searchlib/attribute/defines.cpp index cbe69920ac3..9af6da89662 100644 --- a/searchlib/src/vespa/searchlib/attribute/defines.cpp +++ b/searchlib/src/vespa/searchlib/attribute/defines.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "defines.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.defines"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/dociditerator.cpp b/searchlib/src/vespa/searchlib/attribute/dociditerator.cpp index f7429070f34..400676cece6 100644 --- a/searchlib/src/vespa/searchlib/attribute/dociditerator.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dociditerator.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dociditerator.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.dociditerator"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/dociditerator.h b/searchlib/src/vespa/searchlib/attribute/dociditerator.h index dcce2b150c8..6a6ea3386aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/dociditerator.h +++ b/searchlib/src/vespa/searchlib/attribute/dociditerator.h @@ -63,7 +63,7 @@ protected: template <> inline int32_t -DocIdIterator<AttributePosting>::getData(void) const +DocIdIterator<AttributePosting>::getData() const { return 1; // default weight 1 for single value attributes } @@ -78,20 +78,16 @@ template <typename P> class DocIdMinMaxIterator : public DocIdIterator<P> { public: - DocIdMinMaxIterator(void) + DocIdMinMaxIterator() : DocIdIterator<P>() { } - - inline btree::MinMaxAggregated - getAggregated() const { - return btree::MinMaxAggregated(1, 1); - } + inline btree::MinMaxAggregated getAggregated() const { return btree::MinMaxAggregated(1, 1); } }; template<> inline btree::MinMaxAggregated -DocIdMinMaxIterator<AttributeWeightPosting>::getAggregated(void) const +DocIdMinMaxIterator<AttributeWeightPosting>::getAggregated() const { btree::MinMaxAggregated a; for (const AttributeWeightPosting *cur = _cur, *end = _end; cur != end; ++cur) { diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.cpp index 396c50eba65..34f2c63eb00 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumattribute.h" #include "enumattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.enumattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index ad5e833d3a1..39ee0c4cc63 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -63,7 +63,7 @@ protected: void fillEnum(LoadedVector & loaded) override; void fillEnum0(const void *src, size_t srcLen, EnumIndexVector &eidxs) override; void fixupEnumRefCounts(const EnumVector &enumHist) override; - uint64_t getUniqueValueCount(void) const override; + uint64_t getUniqueValueCount() const override; static EnumType getDefaultEnumTypeValue() { return B::defaultValue(); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 0ed05c35b6f..7127acb8123 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -75,7 +75,7 @@ EnumAttribute<B>::fixupEnumRefCounts( template <typename B> uint64_t -EnumAttribute<B>::getUniqueValueCount(void) const +EnumAttribute<B>::getUniqueValueCount() const { return _enumStore.getNumUniques(); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index 37c66213134..a276beee2d7 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumattributesaver.h" #include "iattributesavetarget.h" #include <vespa/searchlib/util/bufferwriter.h> diff --git a/searchlib/src/vespa/searchlib/attribute/enumcomparator.cpp b/searchlib/src/vespa/searchlib/attribute/enumcomparator.cpp index fde5666371b..564e08d7ee0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumcomparator.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumcomparator.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumcomparator.h" -#include <vespa/searchlib/util/foldedstringcompare.h> #include "enumstore.hpp" +#include <vespa/searchlib/util/foldedstringcompare.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h index 5481e967f24..e3e3a57128e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumcomparator.h +++ b/searchlib/src/vespa/searchlib/attribute/enumcomparator.h @@ -75,29 +75,14 @@ public: **/ EnumStoreFoldedComparatorT(const EnumStoreType & enumStore, EntryValue value, bool prefix = false); - - inline bool - getUsePrefix(void) const - { - return false; - } - - static int - compareFolded(EntryValue lhs, EntryValue rhs) - { - return ParentType::compare(lhs, rhs); - } - - static int - compareFoldedPrefix(EntryValue lhs, EntryValue rhs, size_t prefixLen) - { + inline bool getUsePrefix() const { return false; } + static int compareFolded(EntryValue lhs, EntryValue rhs) { return ParentType::compare(lhs, rhs); } + static int compareFoldedPrefix(EntryValue lhs, EntryValue rhs, size_t prefixLen) { (void) prefixLen; return ParentType::compare(lhs, rhs); } - bool - operator() (const EnumIndex & lhs, const EnumIndex & rhs) const override - { + bool operator() (const EnumIndex & lhs, const EnumIndex & rhs) const override { if (getUsePrefix()) return compareFoldedPrefix(getValue(lhs), getValue(rhs), _prefixLen) < 0; @@ -170,7 +155,7 @@ compareFoldedPrefix(EntryValue lhs, EntryValue rhs, size_t prefixLen); template <> inline bool -EnumStoreFoldedComparatorT<StringEntryType>::getUsePrefix(void) const +EnumStoreFoldedComparatorT<StringEntryType>::getUsePrefix() const { return _prefix; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp index 3b01d02b166..154e9640db3 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumhintsearchcontext.h" #include <vespa/searchlib/queryeval/emptysearch.h> -namespace search -{ +namespace search { using queryeval::SearchIterator; -namespace attribute -{ +namespace attribute { using btree::BTreeNode; using fef::TermFieldMatchData; @@ -28,7 +25,7 @@ EnumHintSearchContext(const EnumStoreDictBase &dictionary, } -EnumHintSearchContext::~EnumHintSearchContext(void) +EnumHintSearchContext::~EnumHintSearchContext() { } @@ -67,7 +64,7 @@ EnumHintSearchContext::createPostingIterator(TermFieldMatchData *matchData, unsigned int -EnumHintSearchContext::approximateHits(void) const +EnumHintSearchContext::approximateHits() const { return (_uniqueValues == 0u) ? 0u diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h index b77db84b520..c573a5818eb 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h @@ -30,7 +30,7 @@ protected: EnumHintSearchContext(const EnumStoreDictBase &dictionary, uint32_t docIdLimit, uint64_t numValues); - ~EnumHintSearchContext(void); + ~EnumHintSearchContext(); void lookupTerm(const EnumStoreComparator &comp); void lookupRange(const EnumStoreComparator &low, const EnumStoreComparator &high); @@ -39,7 +39,7 @@ protected: createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; void fetchPostings(bool strict) override; - unsigned int approximateHits(void) const override; + unsigned int approximateHits() const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index cc50dc7b97d..3b4a9761110 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "enumstore.h" #include "enumstore.hpp" #include <iomanip> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 700cf00fd13..fecd566eadc 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -30,12 +30,7 @@ public: typedef T Type; static uint32_t size(Type) { return fixedSize(); } static uint32_t fixedSize() { return sizeof(T); } - - static bool - hasFold(void) - { - return false; - } + static bool hasFold() { return false; } }; /** @@ -47,12 +42,7 @@ public: typedef const char * Type; static uint32_t size(Type value) { return strlen(value) + fixedSize(); } static uint32_t fixedSize() { return 1; } - - static bool - hasFold(void) - { - return true; - } + static bool hasFold() { return true; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 80872556b0a..ffc21007e95 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -319,7 +319,7 @@ EnumStoreDictBase::EnumStoreDictBase(EnumStoreBase &enumStore) } -EnumStoreDictBase::~EnumStoreDictBase(void) +EnumStoreDictBase::~EnumStoreDictBase() { } @@ -332,14 +332,14 @@ EnumStoreDict<Dictionary>::EnumStoreDict(EnumStoreBase &enumStore) } template <typename Dictionary> -EnumStoreDict<Dictionary>::~EnumStoreDict(void) +EnumStoreDict<Dictionary>::~EnumStoreDict() { } template <typename Dictionary> void -EnumStoreDict<Dictionary>::freezeTree(void) +EnumStoreDict<Dictionary>::freezeTree() { _dict.getAllocator().freeze(); } @@ -361,7 +361,7 @@ EnumStoreDict<Dictionary>::getTreeMemoryUsage() const template <typename Dictionary> void -EnumStoreDict<Dictionary>::reEnumerate(void) +EnumStoreDict<Dictionary>::reEnumerate() { _enumStore.reEnumerate(_dict); } @@ -505,7 +505,7 @@ EnumStoreDict<Dictionary>::findFrozenIndex(const EnumStoreComparator &cmp, template <typename Dictionary> void -EnumStoreDict<Dictionary>::onReset(void) +EnumStoreDict<Dictionary>::onReset() { _dict.clear(); } @@ -529,7 +529,7 @@ EnumStoreDict<Dictionary>::onTrimHoldLists(generation_t firstUsed) template <typename Dictionary> BTreeNode::Ref -EnumStoreDict<Dictionary>::getFrozenRootRef(void) const +EnumStoreDict<Dictionary>::getFrozenRootRef() const { return _dict.getFrozenView().getRoot(); } @@ -570,7 +570,7 @@ lookupFrozenRange(BTreeNode::Ref frozenRootRef, template <> EnumPostingTree & -EnumStoreDict<EnumTree>::getPostingDictionary(void) +EnumStoreDict<EnumTree>::getPostingDictionary() { abort(); } @@ -578,7 +578,7 @@ EnumStoreDict<EnumTree>::getPostingDictionary(void) template <> EnumPostingTree & -EnumStoreDict<EnumPostingTree>::getPostingDictionary(void) +EnumStoreDict<EnumPostingTree>::getPostingDictionary() { return _dict; } @@ -586,7 +586,7 @@ EnumStoreDict<EnumPostingTree>::getPostingDictionary(void) template <> const EnumPostingTree & -EnumStoreDict<EnumTree>::getPostingDictionary(void) const +EnumStoreDict<EnumTree>::getPostingDictionary() const { abort(); } @@ -594,7 +594,7 @@ EnumStoreDict<EnumTree>::getPostingDictionary(void) const template <> const EnumPostingTree & -EnumStoreDict<EnumPostingTree>::getPostingDictionary(void) const +EnumStoreDict<EnumPostingTree>::getPostingDictionary() const { return _dict; } @@ -602,7 +602,7 @@ EnumStoreDict<EnumPostingTree>::getPostingDictionary(void) const template <typename Dictionary> bool -EnumStoreDict<Dictionary>::hasData(void) const +EnumStoreDict<Dictionary>::hasData() const { return Dictionary::LeafNodeType::hasData(); } diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp index ee3950aaef4..9adef1a7ccc 100644 --- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fixedsourceselector.h" #include "singlenumericattribute.hpp" diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index bc9ea4b7ebf..dca416fc384 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -1,23 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "flagattribute.h" -#include <vespa/searchlib/attribute/attributeiterators.h> - -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.flagattribute"); +#include "load_utils.hpp" +#include "attributeiterators.h" +#include "multinumericattribute.hpp" -#include <vespa/searchlib/attribute/multinumericattribute.hpp> #include <vespa/searchlib/queryeval/emptysearch.h> -#include "load_utils.hpp" #include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.attribute.flagattribute"); + namespace search { using queryeval::SearchIterator; -namespace -{ +namespace { template <class FA, typename T> class SaveBits diff --git a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp index 2b1d4a01838..04e2fc7e7a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp @@ -1,4 +1,3 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "i_document_weight_attribute.h" diff --git a/searchlib/src/vespa/searchlib/attribute/iattributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/iattributemanager.cpp index 0141534b1ba..f7a04437f15 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/iattributemanager.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "iattributemanager.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.iattributemanager"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp index e7221c0c401..e8d9bbc5423 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "iattributesavetarget.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.iattributesavetarget"); namespace search { IAttributeSaveTarget::~IAttributeSaveTarget() { diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h index 40d02a4b5c9..6f2b90b0c4c 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h @@ -22,7 +22,7 @@ public: IAttributeSaveTarget() : _header() {} void setHeader(const attribute::AttributeHeader & header) { _header = header; } - bool getEnumerated(void) const { return _header.getEnumerated(); } + bool getEnumerated() const { return _header.getEnumerated(); } /** * Setups this saveTarget before any data is written. Returns true diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.cpp index 89927def124..866869a1c33 100644 --- a/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "ipostinglistsearchcontext.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.ipostinglistsearchcontext"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.h index 48f1d1f509a..94af4ac6ce5 100644 --- a/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistsearchcontext.h @@ -2,25 +2,14 @@ #pragma once +#include <memory> -namespace search -{ - -namespace queryeval -{ - -class SearchIterator; -} +namespace search { -namespace fef -{ - -class TermFieldMatchData; -} +namespace queryeval { class SearchIterator; } +namespace fef { class TermFieldMatchData; } - -namespace attribute -{ +namespace attribute { /** @@ -34,29 +23,14 @@ namespace attribute class IPostingListSearchContext { protected: - - IPostingListSearchContext(void) - { - } - - virtual - ~IPostingListSearchContext(void) - { - } + IPostingListSearchContext() { } + virtual ~IPostingListSearchContext() { } public: - virtual void - fetchPostings(bool strict) = 0; - - virtual std::unique_ptr<queryeval::SearchIterator> - createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) = 0; - - virtual unsigned int - approximateHits(void) const = 0; + virtual void fetchPostings(bool strict) = 0; + virtual std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) = 0; + virtual unsigned int approximateHits() const = 0; }; - } // namespace attribute - } // namespace search - diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp index d86e41d332e..bc7b244c544 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "loadedenumvalue.h" #include <vespa/searchlib/common/sort.h> - -namespace search -{ - -namespace attribute -{ +namespace search { +namespace attribute { void sortLoadedByEnum(LoadedEnumAttributeVector &loaded) @@ -23,6 +18,5 @@ sortLoadedByEnum(LoadedEnumAttributeVector &loaded) } } // namespace attribute - } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h index cdbb1034835..a54546696ff 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h @@ -49,7 +49,7 @@ public: } }; - LoadedEnumAttribute(void) + LoadedEnumAttribute() : _enum(0), _docId(0), _weight(1) @@ -65,23 +65,9 @@ public: { } - uint32_t - getEnum(void) const - { - return _enum; - } - - uint32_t - getDocId(void) const - { - return _docId; - } - - int32_t - getWeight(void) const - { - return _weight; - } + uint32_t getEnum() const { return _enum; } + uint32_t getDocId() const { return _docId; } + int32_t getWeight() const { return _weight; } }; typedef vespalib::Array<LoadedEnumAttribute> LoadedEnumAttributeVector; diff --git a/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp index 1fb2434167f..1034c4ecb1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadednumericvalue.cpp @@ -1,14 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "loadednumericvalue.h" -namespace search -{ - -namespace attribute -{ +namespace search { +namespace attribute { template <typename T> void @@ -89,8 +85,6 @@ void sortLoadedByDocId(SequentialReadModifyWriteVector<LoadedNumericValue<double>> & loaded); - } // namespace attribute - } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp index 954e4342466..26905bcfc2f 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "loadedstringvalue.h" using vespalib::Array; using vespalib::alloc::Alloc; namespace search { - namespace attribute { void @@ -43,6 +41,5 @@ sortLoadedByDocId(LoadedStringVectorReal &loaded) } // namespace attribute - } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/loadedvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedvalue.cpp index cf2e611459b..7b11481128e 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedvalue.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "loadedvalue.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.loadedvalue"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index c4499fbe05f..ac2a925acad 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multi_value_mapping.h" #include "multi_value_mapping.hpp" #include "multivalue.h" diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp index b52d2ae2c73..51a54c89522 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_base.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multi_value_mapping_base.h" #include <vespa/searchcommon/common/compaction_strategy.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp index 33588c94694..4d52150d34a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multienumattribute.h" #include "multienumattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multienumattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp index 76917ee883f..4cc0190a2dc 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multienumattributesaver.h" -#include <vespa/searchlib/util/bufferwriter.h> #include "multivalueattributesaverutils.h" #include "multivalue.h" @@ -12,8 +10,7 @@ using search::multivalueattributesaver::WeightWriter; namespace search { -namespace -{ +namespace { /* * Class to write enum indexes mapped over to either enum values diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.cpp index b6021881ee6..7f475943091 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multinumericattribute.h" #include "multinumericattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multinumericattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp index d2ebd95d9f6..4610e2e6708 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multinumericattributesaver.h" -#include <vespa/searchlib/util/bufferwriter.h> #include "multivalueattributesaverutils.h" #include "multivalue.h" +#include <vespa/searchlib/util/bufferwriter.h> using vespalib::GenerationHandler; using search::multivalueattributesaver::CountWriter; @@ -12,8 +11,7 @@ using search::multivalueattributesaver::WeightWriter; namespace search { -namespace -{ +namespace { class DatWriter { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.cpp index 1f62445a2b2..e711f679263 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multinumericenumattribute.h" #include "multinumericenumattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multinumericenumattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.cpp index 277fdafbf42..99aff626365 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multinumericpostattribute.h" #include "multinumericpostattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multinumericpostattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index 365ec8e9981..8a947ade986 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -45,8 +45,7 @@ MultiValueNumericPostingAttribute<B, M>::MultiValueNumericPostingAttribute(const } template <typename B, typename M> -MultiValueNumericPostingAttribute<B, M>:: -~MultiValueNumericPostingAttribute(void) +MultiValueNumericPostingAttribute<B, M>::~MultiValueNumericPostingAttribute() { this->disableFreeLists(); this->disableElemHoldList(); diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index f71489ca150..7148f3db55d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -30,7 +30,7 @@ MultiValueStringAttributeT<B, M>::~MultiValueStringAttributeT() { } template <typename B, typename M> void -MultiValueStringAttributeT<B, M>::freezeEnumDictionary(void) +MultiValueStringAttributeT<B, M>::freezeEnumDictionary() { this->getEnumStore().freezeTree(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp index a447ad1bd8d..fbdb2961e87 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multistringpostattribute.h" #include "multistringpostattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multistringpostattribute"); namespace search { EnumStoreBase::Index diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 3de4d21e6d0..706fb11eb67 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -103,9 +103,7 @@ public: handleFillPostings(loaded); } - attribute::IPostingListAttributeBase * getIPostingListAttributeBase(void) override { - return this; - } + attribute::IPostingListAttributeBase * getIPostingListAttributeBase() override { return this; } void fillPostingsFixupEnum(const LoadedEnumAttributeVector &loaded) override { fillPostingsFixupEnumBase(loaded); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalue.h b/searchlib/src/vespa/searchlib/attribute/multivalue.h index ae8a96ff6dc..061f072b08a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalue.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalue.h @@ -27,7 +27,7 @@ public: bool operator ==(const Value<T> & rhs) const { return _v == rhs._v; } bool operator <(const Value<T> & rhs) const { return _v < rhs._v; } bool operator >(const Value<T> & rhs) const { return _v > rhs._v; } - static bool hasWeight(void) { return false; } + static bool hasWeight() { return false; } static const bool _hasWeight = false; private: @@ -50,7 +50,7 @@ public: bool operator==(const WeightedValue<T> & rhs) const { return _v == rhs._v; } bool operator <(const WeightedValue<T> & rhs) const { return _v < rhs._v; } bool operator >(const WeightedValue<T> & rhs) const { return _v > rhs._v; } - static bool hasWeight(void) { return true; } + static bool hasWeight() { return true; } static const bool _hasWeight = true; private: diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.cpp index 9db883f7ad2..982970f5fbf 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multivalueattribute.h" #include "multivalueattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.multivalueattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index e02a81e44ef..44e65a5f964 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -64,7 +64,7 @@ public: private: int32_t getWeight(DocId doc, uint32_t idx) const override; - uint64_t getTotalValueCount(void) const override; + uint64_t getTotalValueCount() const override; public: void clearDocs(DocId lidLow, DocId lidLimit) override; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 0aa91842550..63853bc92e4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -176,7 +176,7 @@ MultiValueAttribute<B, M>::getValueCount(DocId doc) const template <typename B, typename M> uint64_t -MultiValueAttribute<B, M>::getTotalValueCount(void) const +MultiValueAttribute<B, M>::getTotalValueCount() const { return _mvMapping.getTotalValueCnt(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp index 0d5a7b465c4..141f7ee1b66 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multivalueattributesaver.h" using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp new file mode 100644 index 00000000000..b9308c7e9ab --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "multivalueattributesaverutils.h" +#include <vespa/searchlib/util/bufferwriter.h> + +namespace search::multivalueattributesaver { + +CountWriter::CountWriter(IAttributeSaveTarget &saveTarget) + : _countWriter(saveTarget.idxWriter().allocBufferWriter()), + _cnt(0) +{ + uint32_t initialCount = 0; + _countWriter->write(&initialCount, sizeof(uint32_t)); +} + +CountWriter::~CountWriter() +{ + _countWriter->flush(); +} + +void +CountWriter::writeCount(uint32_t count) { + _cnt += count; + uint32_t cnt32 = static_cast<uint32_t>(_cnt); + _countWriter->write(&cnt32, sizeof(uint32_t)); +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h index 8024d287a0d..fb41b4be053 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h @@ -3,10 +3,10 @@ #pragma once #include "iattributesavetarget.h" +#include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/vespalib/util/arrayref.h> -namespace search { - -namespace multivalueattributesaver { +namespace search::multivalueattributesaver { /* * Class to write to count files for multivalue attributes (.idx suffix). @@ -17,25 +17,10 @@ class CountWriter uint64_t _cnt; public: - CountWriter(IAttributeSaveTarget &saveTarget) - : _countWriter(saveTarget.idxWriter().allocBufferWriter()), - _cnt(0) - { - uint32_t initialCount = 0; - _countWriter->write(&initialCount, sizeof(uint32_t)); - } - - ~CountWriter() - { - _countWriter->flush(); - } + CountWriter(IAttributeSaveTarget &saveTarget); + ~CountWriter(); - void - writeCount(uint32_t count) { - _cnt += count; - uint32_t cnt32 = static_cast<uint32_t>(_cnt); - _countWriter->write(&cnt32, sizeof(uint32_t)); - } + void writeCount(uint32_t count); }; /* @@ -55,11 +40,9 @@ class WeightWriter<true> public: WeightWriter(IAttributeSaveTarget &saveTarget) : _weightWriter(saveTarget.weightWriter().allocBufferWriter()) - { - } + {} - ~WeightWriter() - { + ~WeightWriter() { _weightWriter->flush(); } @@ -80,20 +63,12 @@ template <> class WeightWriter<false> { public: - WeightWriter(IAttributeSaveTarget &) - { - } + WeightWriter(IAttributeSaveTarget &) {} - ~WeightWriter() - { - } + ~WeightWriter() {} template <typename MultiValueT> - void - writeWeights(vespalib::ConstArrayRef<MultiValueT>) { - } + void writeWeights(vespalib::ConstArrayRef<MultiValueT>) {} }; -} // namespace search::multivalueattributesaver - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index a1d2545adb4..c972e25a7cf 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -108,7 +108,7 @@ EnumIndexMapper::map(EnumStoreBase::Index original, const EnumStoreComparator & template <> void -PostingChange<AttributePosting>::removeDups(void) +PostingChange<AttributePosting>::removeDups() { removeDupAdditions(_additions); removeDupRemovals(_removals); @@ -117,7 +117,7 @@ PostingChange<AttributePosting>::removeDups(void) template <> void -PostingChange<AttributeWeightPosting>::removeDups(void) +PostingChange<AttributeWeightPosting>::removeDups() { removeDupAdditions(_additions); removeDupRemovals(_removals); diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.h b/searchlib/src/vespa/searchlib/attribute/postingchange.h index 357704d7851..ff8eb28c5cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.h +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.h @@ -32,7 +32,7 @@ public: return *this; } - void clear(void) { + void clear() { _additions.clear(); _removals.clear(); } @@ -41,7 +41,7 @@ public: * Remove duplicates in additions and removals vectors, since new * posting list tree doesn't support duplicate entries. */ - void removeDups(void); + void removeDups(); void apply(GrowableBitVector &bv); }; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index b816f19ee79..2e5bc9f5bce 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -217,7 +217,7 @@ PostingListAttributeSubBase(AttributeVector &attr, template <typename P, typename LoadedVector, typename LoadedValueType, typename EnumStoreType> PostingListAttributeSubBase<P, LoadedVector, LoadedValueType, EnumStoreType>:: -~PostingListAttributeSubBase(void) +~PostingListAttributeSubBase() { } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index 5212568e411..062b20e5f96 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -55,7 +55,7 @@ protected: virtual void updatePostings(PostingMap & changePost) = 0; void updatePostings(PostingMap &changePost, EnumStoreComparator &cmp); - void clearAllPostings(void); + void clearAllPostings(); void disableFreeLists() { _postingList.disableFreeLists(); } void disableElemHoldList() { _postingList.disableElemHoldList(); } void fillPostingsFixupEnumBase(const LoadedEnumAttributeVector &loaded); @@ -101,7 +101,7 @@ private: public: PostingListAttributeSubBase(AttributeVector &attr, EnumStore &enumStore); - virtual ~PostingListAttributeSubBase(void); + virtual ~PostingListAttributeSubBase(); void handleFillPostings(LoadedVector &loaded); void updatePostings(PostingMap &changePost) override; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 73a5595667e..9665872f8de 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -53,17 +53,17 @@ protected: uint32_t minBvDocFreq, bool useBitVector); - ~PostingListSearchContext(void); + ~PostingListSearchContext(); void lookupTerm(const EnumStoreComparator &comp); void lookupRange(const EnumStoreComparator &low, const EnumStoreComparator &high); - void lookupSingle(void); + void lookupSingle(); virtual bool useThis(const DictionaryConstIterator & it) const { (void) it; return true; } - float calculateFilteringCost(void) const { + float calculateFilteringCost() const { // filtering search time (ms) ~ FSTC * numValues; (FSTC = // Filtering Search Time Constant) return _FSTC * _numValues; @@ -75,13 +75,13 @@ protected: return _PLSTC * approxNumHits; } - uint32_t calculateApproxNumHits(void) const { + uint32_t calculateApproxNumHits() const { float docsPerUniqueValue = static_cast<float>(_docIdLimit) / static_cast<float>(_dictSize); return static_cast<uint32_t>(docsPerUniqueValue * _uniqueValues); } - virtual bool fallbackToFiltering(void) const { + virtual bool fallbackToFiltering() const { uint32_t numHits = calculateApproxNumHits(); // numHits > 1000: make sure that posting lists are unit tested. return (numHits > 1000) && @@ -127,10 +127,10 @@ protected: bool useBitVector); ~PostingListSearchContextT(); - void lookupSingle(void); - size_t countHits(void) const; + void lookupSingle(); + size_t countHits() const; void fillArray(size_t numDocs); - void fillBitVector(void); + void fillBitVector(); PostingVector & merge(PostingVector &v, PostingVector &temp, @@ -145,8 +145,8 @@ protected: queryeval::SearchIterator::UP createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; - unsigned int singleHits(void) const; - unsigned int approximateHits(void) const override; + unsigned int singleHits() const; + unsigned int approximateHits() const override; void applyRangeLimit(int rangeLimit); }; @@ -174,7 +174,7 @@ protected: uint32_t minBvCocFreq, bool useBitVector); - unsigned int approximateHits(void) const override; + unsigned int approximateHits() const override; }; @@ -241,12 +241,12 @@ private: void getIterators(bool shouldApplyRangeLimit); bool valid() const override { return this->isValid(); } - bool fallbackToFiltering(void) const override { + bool fallbackToFiltering() const override { return (this->getRangeLimit() != 0) ? false : Parent::fallbackToFiltering(); } - unsigned int approximateHits(void) const override { + unsigned int approximateHits() const override { const unsigned int estimate = PostingListSearchContextT<DataT>::approximateHits(); const unsigned int limit = std::abs(this->getRangeLimit()); return ((limit > 0) && (limit < estimate)) diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 693614110b8..270ae161f4c 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -41,7 +41,7 @@ PostingListSearchContextT<DataT>::~PostingListSearchContextT() {} template <typename DataT> void -PostingListSearchContextT<DataT>::lookupSingle(void) +PostingListSearchContextT<DataT>::lookupSingle() { PostingListSearchContext::lookupSingle(); if (!_pidx.valid()) @@ -82,7 +82,7 @@ PostingListSearchContextT<DataT>::lookupSingle(void) template <typename DataT> size_t -PostingListSearchContextT<DataT>::countHits(void) const +PostingListSearchContextT<DataT>::countHits() const { size_t sum(0); for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { @@ -122,7 +122,7 @@ PostingListSearchContextT<DataT>::fillArray(size_t numDocs) template <typename DataT> void -PostingListSearchContextT<DataT>::fillBitVector(void) +PostingListSearchContextT<DataT>::fillBitVector() { _bitVector = BitVector::create(_docIdLimit); BitVector &bv(*_bitVector); @@ -272,7 +272,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) template <typename DataT> unsigned int -PostingListSearchContextT<DataT>::singleHits(void) const +PostingListSearchContextT<DataT>::singleHits() const { if (_gbv) { // Some inaccuracy is expected, data changes underfeet @@ -290,7 +290,7 @@ PostingListSearchContextT<DataT>::singleHits(void) const template <typename DataT> unsigned int -PostingListSearchContextT<DataT>::approximateHits(void) const +PostingListSearchContextT<DataT>::approximateHits() const { unsigned int numHits = 0; if (_uniqueValues == 0u) { @@ -362,7 +362,7 @@ PostingListFoldedSearchContextT(const Dictionary &dictionary, template <typename DataT> unsigned int -PostingListFoldedSearchContextT<DataT>::approximateHits(void) const +PostingListFoldedSearchContextT<DataT>::approximateHits() const { unsigned int numHits = 0; if (_uniqueValues == 0u) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp index 38bf41d4c49..1e10a87a5a7 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "postinglisttraits.h" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.postinglisttraits"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 9e9268b0707..a21884bca4a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -1,17 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "postingstore.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/attribute/status.h> -namespace search -{ +namespace search { -namespace attribute -{ +namespace attribute { using btree::BTreeNoLeafData; diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 483468c5d91..7d4d92097b8 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -24,7 +24,7 @@ public: return (_idxFileSize - _idxHeaderLen) /sizeof(uint32_t); } - size_t getEnumCount(void) const { + size_t getEnumCount() const { size_t dataSize(_datFileSize - _datHeaderLen); assert((dataSize % sizeof(uint32_t)) == 0); return dataSize / sizeof(uint32_t); diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 8909b854423..2e34638a420 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -1,16 +1,15 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "reference_attribute.h" +#include "reference_attribute_saver.h" #include "attributesaver.h" -#include <vespa/vespalib/data/fileheader.h> #include "readerbase.h" #include <vespa/searchlib/datastore/unique_store_builder.h> #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/searchlib/datastore/unique_store.hpp> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper.h> -#include "reference_attribute_saver.h" +#include <vespa/vespalib/data/fileheader.h> namespace search { namespace attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index fe987a35d60..be90f617e10 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singleenumattribute.h" #include "singleenumattribute.hpp" -namespace search -{ +namespace search { using attribute::Config; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 4b19f8d1af0..0624f9afc02 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -282,7 +282,7 @@ SingleValueEnumAttribute<B>::clearDocs(DocId lidLow, DocId lidLimit) template <typename B> void -SingleValueEnumAttribute<B>::onShrinkLidSpace(void) +SingleValueEnumAttribute<B>::onShrinkLidSpace() { EnumHandle e; bool findDefaultEnumRes(this->findEnum(this->getDefaultEnumTypeValue(), e)); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.cpp index b08931f36fc..17692734bdc 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlenumericattribute.h" #include "singlenumericattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.singlenumericattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 4285c868f24..f7e3fe79e5d 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -22,7 +22,7 @@ SingleValueNumericAttribute(const vespalib::string & baseFileName, const Attribu { } template <typename B> -SingleValueNumericAttribute<B>::~SingleValueNumericAttribute(void) +SingleValueNumericAttribute<B>::~SingleValueNumericAttribute() { getGenerationHolder().clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp index fe8cdf1ca42..01c2722d946 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlenumericattributesaver.h" #include "iattributesavetarget.h" @@ -8,8 +7,7 @@ using vespalib::GenerationHandler; namespace search { -namespace -{ +namespace { const uint32_t MIN_ALIGNMENT = 4096; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.cpp index df86159833d..ae09d08b388 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlenumericenumattribute.h" #include "singlenumericenumattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.singlenumericenumattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.cpp index 3eb6f61101d..3fb8b758f12 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlenumericpostattribute.h" #include "singlenumericpostattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.singlenumericpostattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 83e9d6fdc92..4ac06de48e8 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -33,7 +33,7 @@ SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, } -SingleValueSmallNumericAttribute::~SingleValueSmallNumericAttribute(void) +SingleValueSmallNumericAttribute::~SingleValueSmallNumericAttribute() { getGenerationHolder().clearHoldLists(); } @@ -253,11 +253,11 @@ namespace template <typename TT> uint32_t -log2bits(void); +log2bits(); template <> uint32_t -log2bits<uint32_t>(void) +log2bits<uint32_t>() { return 0x05u; } diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.cpp index 0c6c6d8ee06..e7fa5962ca7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlestringattribute.h" #include "singlestringattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.singlestringattribute"); namespace search { template class SingleValueStringAttributeT<EnumAttribute<StringAttribute>>; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 4c8587791c4..46ab73e07db 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -29,7 +29,7 @@ SingleValueStringAttributeT<B>::~SingleValueStringAttributeT() { } template <typename B> void -SingleValueStringAttributeT<B>::freezeEnumDictionary(void) { +SingleValueStringAttributeT<B>::freezeEnumDictionary() { this->getEnumStore().freezeTree(); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.cpp index e0ac10c10af..cef98c431bd 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "singlestringpostattribute.h" #include "singlestringpostattribute.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.singlestringpostattribute"); namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 2130fcafc5c..622f02675e8 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -347,7 +347,7 @@ readHeader(const vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -FeatureDecodeContext<bigEndian>::getIdentifier(void) const +FeatureDecodeContext<bigEndian>::getIdentifier() const { return noFeatures; } @@ -423,7 +423,7 @@ writeHeader(vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -FeatureEncodeContext<bigEndian>::getIdentifier(void) const +FeatureEncodeContext<bigEndian>::getIdentifier() const { return noFeatures; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 83133e53c93..3f18de0a4b4 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -1136,7 +1136,7 @@ public: // minus sizeof uint64_t. Then shifted left by 3 to represent bits. uint64_t _fileReadBias; - DecodeContext64Base(void) + DecodeContext64Base() : search::ComprFileDecodeContext(), _valI(NULL), _valE(NULL), @@ -1167,7 +1167,7 @@ public: } virtual - ~DecodeContext64Base(void) + ~DecodeContext64Base() { } @@ -1261,11 +1261,11 @@ public: _valE += END_BUFFER_SAFETY; } - const uint64_t *getCompr(void) const { + const uint64_t *getCompr() const { return (_preRead == 0) ? (_valI - 1) : (_valI - 2); } - int getBitOffset(void) const { + int getBitOffset() const { return (_preRead == 0) ? 0 : 64 - _preRead; } @@ -1290,7 +1290,7 @@ private: public: typedef EncodeContext64<bigEndian> EC; - DecodeContext64(void) + DecodeContext64() : DecodeContext64Base() { } @@ -1415,7 +1415,7 @@ public: * Used by iterators when switching from bitwise to bytewise decoding. */ const uint8_t * - getByteCompr(void) const + getByteCompr() const { assert((_preRead & 7) == 0); return reinterpret_cast<const uint8_t *>(getCompr()) + @@ -1501,7 +1501,7 @@ public: using ParentClass::readBits; using ParentClass::ReadBits; - FeatureDecodeContext(void) + FeatureDecodeContext() : ParentClass(), _readContext(NULL) { @@ -1537,19 +1537,19 @@ public: } search::ComprFileReadContext * - getReadContext(void) const + getReadContext() const { return _readContext; } void - readComprBuffer(void) + readComprBuffer() { _readContext->readComprBuffer(); } void - readComprBufferIfNeeded(void) + readComprBufferIfNeeded() { if (__builtin_expect(_valI >= _valE, false)) readComprBuffer(); @@ -1566,7 +1566,7 @@ public: const vespalib::string &prefix); virtual const vespalib::string & - getIdentifier(void) const; + getIdentifier() const; virtual void readFeatures(DocIdAndFeatures &features); @@ -1641,7 +1641,7 @@ public: using ParentClass::smallPadBits; public: - FeatureEncodeContext(void) + FeatureEncodeContext() : ParentClass(), _writeContext(NULL) { diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index ae6254e44e9..6acfa022a14 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -2,17 +2,14 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include "compression.h" #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/vespalib/objects/nbostream.h> -namespace search -{ +namespace search { -namespace bitcompression -{ +namespace bitcompression { using vespalib::nbostream; diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h index c3961f7fcbd..974f4bb522f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.h @@ -23,7 +23,7 @@ public: uint64_t _numWordIds; // Number of words in dictionary uint64_t _minWordNum; // Minimum word number - PostingListCountFileDecodeContext(void) + PostingListCountFileDecodeContext() : ParentClass(), _avgBitsPerDoc(10), _minChunkDocs(262144), @@ -57,7 +57,7 @@ public: uint64_t _numWordIds; // Number of words in dictionary uint64_t _minWordNum; // Mininum word number - PostingListCountFileEncodeContext(void) + PostingListCountFileEncodeContext() : ParentClass(), _avgBitsPerDoc(10), _minChunkDocs(262144), @@ -72,7 +72,7 @@ public: void writeCounts(const PostingListCounts &counts); void writeWordNum(uint64_t wordNum); - static uint64_t noWordNum(void) { + static uint64_t noWordNum() { return std::numeric_limits<uint64_t>::max(); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index 2fc1a1a071b..e5af0580b09 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -246,7 +246,7 @@ PageDict4SSWriter::PageDict4SSWriter(SSEC &sse) { } -PageDict4SSWriter::~PageDict4SSWriter(void) +PageDict4SSWriter::~PageDict4SSWriter() { } @@ -355,7 +355,7 @@ addOverflowCounts(const vespalib::stringref &word, void -PageDict4SSWriter::flush(void) +PageDict4SSWriter::flush() { } @@ -454,13 +454,13 @@ PageDict4SPWriter::setup() } -PageDict4SPWriter::~PageDict4SPWriter(void) +PageDict4SPWriter::~PageDict4SPWriter() { } void -PageDict4SPWriter::flushPage(void) +PageDict4SPWriter::flushPage() { assert(_l3Entries > 0); assert(_l3Size > 0); @@ -523,7 +523,7 @@ PageDict4SPWriter::flushPage(void) void -PageDict4SPWriter::flush(void) +PageDict4SPWriter::flush() { if (!empty()) { flushPage(); @@ -537,7 +537,7 @@ PageDict4SPWriter::flush(void) void -PageDict4SPWriter::resetPage(void) +PageDict4SPWriter::resetPage() { _eL3.setupWrite(_wcL3); _eL4.setupWrite(_wcL4); @@ -837,13 +837,13 @@ PageDict4PWriter::setup() } -PageDict4PWriter::~PageDict4PWriter(void) +PageDict4PWriter::~PageDict4PWriter() { } void -PageDict4PWriter::flushPage(void) +PageDict4PWriter::flushPage() { assert(_countsEntries > 0); assert(_countsSize > 0); @@ -904,7 +904,7 @@ PageDict4PWriter::flushPage(void) void -PageDict4PWriter::flush(void) +PageDict4PWriter::flush() { if (!empty()) { flushPage(); @@ -918,7 +918,7 @@ PageDict4PWriter::flush(void) void -PageDict4PWriter::resetPage(void) +PageDict4PWriter::resetPage() { _eCounts.setupWrite(_wcCounts); _eL1.setupWrite(_wcL1); @@ -1195,7 +1195,7 @@ PageDict4PWriter::checkPointRead(vespalib::nbostream &in) PageDict4SSLookupRes:: -PageDict4SSLookupRes(void) +PageDict4SSLookupRes() : _l6Word(), _lastWord(), _l6StartOffset(), @@ -1211,7 +1211,7 @@ PageDict4SSLookupRes(void) PageDict4SSLookupRes:: -~PageDict4SSLookupRes(void) +~PageDict4SSLookupRes() { } @@ -1246,7 +1246,7 @@ PageDict4SSReader(ComprBuffer &cb, PageDict4SSReader:: -~PageDict4SSReader(void) +~PageDict4SSReader() { } @@ -1689,7 +1689,7 @@ PageDict4SSReader::checkPointRead(vespalib::nbostream &in) PageDict4SPLookupRes:: -PageDict4SPLookupRes(void) +PageDict4SPLookupRes() : _l3Word(), _lastWord(), _l3StartOffset(), @@ -1700,7 +1700,7 @@ PageDict4SPLookupRes(void) PageDict4SPLookupRes:: -~PageDict4SPLookupRes(void) +~PageDict4SPLookupRes() { } @@ -1888,7 +1888,7 @@ lookup(const SSReader &ssReader, PageDict4PLookupRes:: -PageDict4PLookupRes(void) +PageDict4PLookupRes() : _counts(), _startOffset(), _wordNum(1u), @@ -1899,7 +1899,7 @@ PageDict4PLookupRes(void) PageDict4PLookupRes:: -~PageDict4PLookupRes(void) +~PageDict4PLookupRes() { } @@ -2132,13 +2132,13 @@ PageDict4Reader::setup() } -PageDict4Reader::~PageDict4Reader(void) +PageDict4Reader::~PageDict4Reader() { } void -PageDict4Reader::setupPage(void) +PageDict4Reader::setupPage() { #if 0 LOG(info, @@ -2224,7 +2224,7 @@ PageDict4Reader::setupPage(void) void -PageDict4Reader::setupSPage(void) +PageDict4Reader::setupSPage() { #if 0 LOG(info, "setupSPage(%d),", (int) _spd.getReadOffset()); diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h index f4012a2644b..e974e546956 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h @@ -19,7 +19,7 @@ public: uint64_t _fileOffset; uint64_t _accNumDocs; - PageDict4StartOffset(void) + PageDict4StartOffset() : _fileOffset(0u), _accNumDocs(0u) { @@ -86,91 +86,21 @@ public: using Counts = index::PostingListCounts; typedef PageDict4StartOffset StartOffset; - static uint32_t - getPageByteSize(void) - { - return 4096; - } - - static uint32_t - getPageBitSize(void) - { - return getPageByteSize() * 8; - } - - static uint32_t - getPageHeaderBitSize(void) - { - return 15u + 15u + 15u + 12u; - } - - static uint32_t - getMaxFileHeaderPad(void) - { - return 999u; - } - - static uint32_t - getFileHeaderPad(uint32_t offset); - - static uint32_t - getL1SkipStride(void) - { - return 16; - } - - static uint32_t - getL2SkipStride(void) - { - return 8; - } - - static uint32_t - getL4SkipStride(void) - { - return 16; - } - - static uint32_t - getL5SkipStride(void) - { - return 8; - } - - static uint32_t - getL7SkipStride(void) - { - return 8; - } - - static uint32_t - noL7Ref(void) - { - return std::numeric_limits<uint32_t>::max(); - } - - static uint32_t - getL1Entries(uint32_t countsEntries) - { - return (countsEntries - 1) / getL1SkipStride(); - } - - static uint32_t - getL2Entries(uint32_t l1Entries) - { - return l1Entries / getL2SkipStride(); - } - - static uint32_t - getL4Entries(uint32_t l3Entries) - { - return (l3Entries - 1) / getL4SkipStride(); - } - - static uint32_t - getL5Entries(uint32_t l4Entries) - { - return l4Entries / getL5SkipStride(); + static uint32_t getPageByteSize() { return 4096; } + static uint32_t getPageBitSize() { return getPageByteSize() * 8; } + static uint32_t getPageHeaderBitSize() { return 15u + 15u + 15u + 12u; } + static uint32_t getMaxFileHeaderPad() { return 999u; } + static uint32_t getFileHeaderPad(uint32_t offset); + static uint32_t getL1SkipStride() { return 16; } + static uint32_t getL2SkipStride() { return 8; } + static uint32_t getL4SkipStride() { return 16; } + static uint32_t getL5SkipStride() { return 8; } + static uint32_t getL7SkipStride() { return 8; } + static uint32_t noL7Ref() { return std::numeric_limits<uint32_t>::max(); } + static uint32_t getL1Entries(uint32_t countsEntries) { return (countsEntries - 1) / getL1SkipStride(); } + static uint32_t getL2Entries(uint32_t l1Entries) { return l1Entries / getL2SkipStride(); } + static uint32_t getL4Entries(uint32_t l3Entries) { return (l3Entries - 1) / getL4SkipStride(); } + static uint32_t getL5Entries(uint32_t l4Entries) { return l4Entries / getL5SkipStride(); } }; /* @@ -203,7 +133,7 @@ private: public: PageDict4SSWriter(SSEC &sse); - ~PageDict4SSWriter(void); + ~PageDict4SSWriter(); /* * Add L6 skip entry. @@ -229,7 +159,7 @@ public: uint64_t wordNum); void - flush(void); + flush(); void @@ -325,19 +255,19 @@ public: PageDict4SPWriter(SSWriter &sparseSparsewriter, EC &spe); - ~PageDict4SPWriter(void); + ~PageDict4SPWriter(); void setup(); void - flushPage(void); + flushPage(); void - flush(void); + flush(); void - resetPage(void); + resetPage(); void addL3Skip(const vespalib::stringref &word, @@ -353,13 +283,13 @@ public: addL5Skip(size_t &lcp); bool - empty(void) const + empty() const { return _l3Entries == 0; } uint32_t - getSparsePageNum(void) const + getSparsePageNum() const { return _sparsePageNum; } @@ -476,19 +406,19 @@ public: PageDict4PWriter(SPWriter &spWriter, EC &pe); - ~PageDict4PWriter(void); + ~PageDict4PWriter(); void setup(); void - flushPage(void); + flushPage(); void - flush(void); + flush(); void - resetPage(void); + resetPage(); void addCounts(const vespalib::stringref &word, @@ -501,13 +431,13 @@ public: addL2Skip(size_t &lcp); bool - empty(void) const + empty() const { return _countsEntries == 0; } uint64_t - getPageNum(void) const + getPageNum() const { return _pageNum; } @@ -543,9 +473,9 @@ public: bool _res; bool _overflow; - PageDict4SSLookupRes(void); + PageDict4SSLookupRes(); - ~PageDict4SSLookupRes(void); + ~PageDict4SSLookupRes(); }; /* Reader for sparse sparse file. @@ -671,7 +601,7 @@ public: uint32_t pFileHeaderSize, uint64_t pFileBitLen); - ~PageDict4SSReader(void); + ~PageDict4SSReader(); void setup(DC &ssd); @@ -683,7 +613,7 @@ public: lookupOverflow(uint64_t wordNum) const; const DC & - getSSD(void) const + getSSD() const { return _ssd; } @@ -710,9 +640,9 @@ public: uint64_t _l3WordNum; public: - PageDict4SPLookupRes(void); + PageDict4SPLookupRes(); - ~PageDict4SPLookupRes(void); + ~PageDict4SPLookupRes(); void lookup(const SSReader &ssReader, @@ -741,9 +671,9 @@ public: vespalib::string *_nextWord; public: - PageDict4PLookupRes(void); + PageDict4PLookupRes(); - ~PageDict4PLookupRes(void); + ~PageDict4PLookupRes(); bool lookup(const SSReader &ssReader, @@ -794,7 +724,7 @@ public: DC &spd, DC &pd); - ~PageDict4Reader(void); + ~PageDict4Reader(); void setup(); diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index d209c99946b..f50be0d5119 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -2,7 +2,6 @@ // Copyright (C) 2002-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include "compression.h" #include "posocccompression.h" #include <vespa/searchlib/index/schemautil.h> @@ -10,6 +9,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> + #include <vespa/log/log.h> LOG_SETUP(".posocccompression"); @@ -42,7 +42,7 @@ namespace search { namespace bitcompression { -PosOccFieldParams::PosOccFieldParams(void) +PosOccFieldParams::PosOccFieldParams() : _elemLenK(0), _hasElements(false), _hasElementWeights(false), @@ -217,7 +217,7 @@ PosOccFieldParams::writeHeader(vespalib::GenericHeader &header, } -PosOccFieldsParams::PosOccFieldsParams(void) +PosOccFieldsParams::PosOccFieldsParams() : _numFields(0u), _fieldParams(NULL), _params() @@ -710,7 +710,7 @@ readHeader(const vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -EG2PosOccDecodeContext<bigEndian>::getIdentifier(void) const +EG2PosOccDecodeContext<bigEndian>::getIdentifier() const { return EG64PosOccId2; } @@ -728,7 +728,7 @@ writeHeader(vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -EG2PosOccEncodeContext<bigEndian>::getIdentifier(void) const +EG2PosOccEncodeContext<bigEndian>::getIdentifier() const { return EG64PosOccId2; } @@ -1205,7 +1205,7 @@ readHeader(const vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -EGPosOccDecodeContext<bigEndian>::getIdentifier(void) const +EGPosOccDecodeContext<bigEndian>::getIdentifier() const { return EG64PosOccId; } @@ -1223,7 +1223,7 @@ writeHeader(vespalib::GenericHeader &header, template <bool bigEndian> const vespalib::string & -EGPosOccEncodeContext<bigEndian>::getIdentifier(void) const +EGPosOccEncodeContext<bigEndian>::getIdentifier() const { return EG64PosOccId; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h index acc0e9f94ce..c2b6c5a062f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h @@ -134,7 +134,7 @@ public: } uint32_t getNumFields() const { return _numFields; } - const PosOccFieldParams *getFieldParams(void) const { return _fieldParams; } + const PosOccFieldParams *getFieldParams() const { return _fieldParams; } void getParams(PostingListParams ¶ms) const; void setParams(const PostingListParams ¶ms); void setSchemaParams(const Schema &schema, const uint32_t indexId); @@ -195,7 +195,7 @@ public: } void readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix) override; - const vespalib::string &getIdentifier(void) const override; + const vespalib::string &getIdentifier() const override; void readFeatures(search::index::DocIdAndFeatures &features) override; void skipFeatures(unsigned int count) override; void unpackFeatures(const search::fef::TermFieldMatchDataArray &matchData, uint32_t docId) override; @@ -290,7 +290,7 @@ public: void readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix) override; void writeHeader(vespalib::GenericHeader &header, const vespalib::string &prefix) const override; - const vespalib::string &getIdentifier(void) const override; + const vespalib::string &getIdentifier() const override; void writeFeatures(const DocIdAndFeatures &features) override; void setParams(const PostingListParams ¶ms) override; void getParams(PostingListParams ¶ms) const override; @@ -347,7 +347,7 @@ public: } void readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix) override; - const vespalib::string &getIdentifier(void) const override; + const vespalib::string &getIdentifier() const override; void readFeatures(search::index::DocIdAndFeatures &features) override; void skipFeatures(unsigned int count) override; void unpackFeatures(const search::fef::TermFieldMatchDataArray &matchData, uint32_t docId) override; @@ -441,7 +441,7 @@ public: void readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix) override; void writeHeader(vespalib::GenericHeader &header, const vespalib::string &prefix) const override; - const vespalib::string &getIdentifier(void) const override; + const vespalib::string &getIdentifier() const override; void writeFeatures(const DocIdAndFeatures &features) override; void setParams(const PostingListParams ¶ms) override; void getParams(PostingListParams ¶ms) const override; diff --git a/searchlib/src/vespa/searchlib/btree/btree.h b/searchlib/src/vespa/searchlib/btree/btree.h index 7f96e3647e2..122bcf781ef 100644 --- a/searchlib/src/vespa/searchlib/btree/btree.h +++ b/searchlib/src/vespa/searchlib/btree/btree.h @@ -138,7 +138,7 @@ public: } const AggrT & - getAggregated(void) const + getAggregated() const { return _tree.getAggregated(_alloc); } diff --git a/searchlib/src/vespa/searchlib/btree/btreeaggregator.cpp b/searchlib/src/vespa/searchlib/btree/btreeaggregator.cpp index 75e07cd7514..a02e2744861 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeaggregator.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreeaggregator.cpp @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreetraits.h" #include "btreeaggregator.hpp" #include "noaggrcalc.h" #include "minmaxaggrcalc.h" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeAggregator<uint32_t, uint32_t>; template class BTreeAggregator<uint32_t, BTreeNoLeafData>; @@ -19,7 +14,4 @@ template class BTreeAggregator<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::LEAF_SLOTS, MinMaxAggrCalc>; - -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreebuilder.cpp b/searchlib/src/vespa/searchlib/btree/btreebuilder.cpp index 4a2096f8fc4..4123f3c1afa 100644 --- a/searchlib/src/vespa/searchlib/btree/btreebuilder.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreebuilder.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreebuilder.h" #include "btreenode.hpp" #include "btreebuilder.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeBuilder<uint32_t, uint32_t, NoAggregated, @@ -25,6 +20,4 @@ template class BTreeBuilder<uint32_t, int32_t, BTreeDefaultTraits::LEAF_SLOTS, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreebuilder.h b/searchlib/src/vespa/searchlib/btree/btreebuilder.h index b68a4e440d5..5f3dc5a43d2 100644 --- a/searchlib/src/vespa/searchlib/btree/btreebuilder.h +++ b/searchlib/src/vespa/searchlib/btree/btreebuilder.h @@ -50,19 +50,19 @@ private: const AggrCalcT &_aggrCalc; void - normalize(void); + normalize(); void - allocNewLeafNode(void); + allocNewLeafNode(); InternalNodeType * - createInternalNode(void); + createInternalNode(); public: BTreeBuilder(NodeAllocatorType &allocator); BTreeBuilder(NodeAllocatorType &allocator, const AggrCalcT &aggrCalc); - ~BTreeBuilder(void); + ~BTreeBuilder(); void recursiveDelete(NodeRef node); @@ -71,13 +71,13 @@ public: insert(const KeyT &key, const DataT &data); NodeRef - handover(void); + handover(); void - reuse(void); + reuse(); void - clear(void); + clear(); }; extern template class BTreeBuilder<uint32_t, uint32_t, diff --git a/searchlib/src/vespa/searchlib/btree/btreebuilder.hpp b/searchlib/src/vespa/searchlib/btree/btreebuilder.hpp index 60e0d143533..f1c59c1b2d3 100644 --- a/searchlib/src/vespa/searchlib/btree/btreebuilder.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreebuilder.hpp @@ -49,7 +49,7 @@ BTreeBuilder(NodeAllocatorType &allocator, const AggrCalcT &aggrCalc) template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -~BTreeBuilder(void) +~BTreeBuilder() { clear(); } @@ -80,7 +80,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> void BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -normalize(void) +normalize() { std::vector<NodeRef> leftInodes; // left to rightmost nodes in tree LeafNodeType *leftLeaf; @@ -302,7 +302,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> void BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -allocNewLeafNode(void) +allocNewLeafNode() { InternalNodeType *inode; NodeRef child; @@ -392,7 +392,7 @@ template <typename KeyT, typename DataT, typename AggrT, typename BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>::NodeRef BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -handover(void) +handover() { NodeRef ret; @@ -417,7 +417,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> void BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -reuse(void) +reuse() { clear(); _leaf = _allocator.allocLeafNode(); @@ -430,7 +430,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> void BTreeBuilder<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>:: -clear(void) +clear() { if (!_inodes.empty()) { recursiveDelete(_inodes.back().ref); diff --git a/searchlib/src/vespa/searchlib/btree/btreeinserter.cpp b/searchlib/src/vespa/searchlib/btree/btreeinserter.cpp index 726d5208e46..ff169c0b6c8 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeinserter.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreeinserter.cpp @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreeinserter.h" #include "btreenodeallocator.h" #include "btreerootbase.hpp" #include "btreeinserter.hpp" #include "btreenode.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeInserter<uint32_t, uint32_t, NoAggregated>; template class BTreeInserter<uint32_t, BTreeNoLeafData, NoAggregated>; @@ -20,6 +15,4 @@ template class BTreeInserter<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreeiterator.cpp b/searchlib/src/vespa/searchlib/btree/btreeiterator.cpp index 48accfd1ed9..1d417ee9fb6 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeiterator.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreeiterator.cpp @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreeroot.h" #include "btreenodeallocator.h" #include "btreeiterator.hpp" #include "btreenode.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeIteratorBase<uint32_t, uint32_t, NoAggregated>; template class BTreeIteratorBase<uint32_t, BTreeNoLeafData, NoAggregated>; @@ -22,6 +17,4 @@ template class BTreeIterator<uint32_t, uint32_t, NoAggregated>; template class BTreeIterator<uint32_t, BTreeNoLeafData, NoAggregated>; template class BTreeIterator<uint32_t, int32_t, MinMaxAggregated>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreeiterator.h b/searchlib/src/vespa/searchlib/btree/btreeiterator.h index 6b631412470..0ec24ae5da8 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeiterator.h +++ b/searchlib/src/vespa/searchlib/btree/btreeiterator.h @@ -41,13 +41,13 @@ class NodeElement uint32_t _idx; NodeType * - getWNode(void) const + getWNode() const { return const_cast<NodeType *>(_node); } public: - NodeElement(void) + NodeElement() : _node(NULL), _idx(0u) { @@ -66,7 +66,7 @@ public: } const NodeType * - getNode(void) const + getNode() const { return _node; } @@ -78,19 +78,19 @@ public: } uint32_t - getIdx(void) const + getIdx() const { return _idx; } void - incIdx(void) + incIdx() { ++_idx; } void - decIdx(void) + decIdx() { --_idx; } @@ -121,7 +121,7 @@ public: } void - adjustLeftVictimKilled(void) + adjustLeftVictimKilled() { assert(_idx > 0); --_idx; @@ -383,7 +383,7 @@ public: * Return if the tree has data or not (e.g. keys and data or only keys). */ static bool - hasData(void) + hasData() { return LeafNodeType::hasData(); } @@ -392,19 +392,19 @@ public: * Move the iterator directly to end. Used by findHelper method in BTree. */ void - setupEnd(void); + setupEnd(); /** * Setup iterator to be empty and not be associated with any tree. */ void - setupEmpty(void); + setupEmpty(); /** * Move iterator to beyond last element in the current tree. */ void - end(void) __attribute__((noinline)); + end() __attribute__((noinline)); /** * Move iterator to beyond last element in the given tree. @@ -418,7 +418,7 @@ public: * Move iterator to first element in the current tree. */ void - begin(void); + begin(); /** * Move iterator to first element in the given tree. @@ -432,13 +432,13 @@ public: * Move iterator to last element in the current tree. */ void - rbegin(void); + rbegin(); /* * Get aggregated values for the current tree. */ const AggrT & - getAggregated(void) const; + getAggregated() const; bool identical(const BTreeIteratorBase &rhs) const; @@ -756,7 +756,7 @@ public: } NodeAllocatorType & - getAllocator(void) const + getAllocator() const { return const_cast<NodeAllocatorType &>(*_allocator); } @@ -808,7 +808,7 @@ private: const AggrCalcT &aggrCalc); LeafNodeType * - getLeafNode(void) const + getLeafNode() const { return _leaf.getWNode(); } @@ -823,13 +823,13 @@ private: } uint32_t - getLeafNodeIdx(void) const + getLeafNodeIdx() const { return _leaf.getIdx(); } uint32_t - getPathSize(void) const + getPathSize() const { return _pathSize; } diff --git a/searchlib/src/vespa/searchlib/btree/btreeiterator.hpp b/searchlib/src/vespa/searchlib/btree/btreeiterator.hpp index ed04a2f94e9..52d59d3932a 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeiterator.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreeiterator.hpp @@ -104,7 +104,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> void BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -setupEnd(void) +setupEnd() { _leaf.setNodeAndIdx(NULL, 0u); } @@ -114,7 +114,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> void BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -setupEmpty(void) +setupEmpty() { clearPath(0u); _leaf.setNodeAndIdx(NULL, 0u); @@ -126,7 +126,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> void BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -end(void) +end() { if (_pathSize == 0) { if (_leafRoot == NULL) @@ -257,7 +257,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> void BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -begin(void) +begin() { uint32_t pidx = _pathSize; if (pidx > 0u) { @@ -320,7 +320,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> void BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -rbegin(void) +rbegin() { uint32_t pidx = _pathSize; if (pidx > 0u) { @@ -353,7 +353,7 @@ template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> const AggrT & BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -getAggregated(void) const +getAggregated() const { // XXX: Undefined behavior if tree is empty. uint32_t pidx = _pathSize; diff --git a/searchlib/src/vespa/searchlib/btree/btreenode.cpp b/searchlib/src/vespa/searchlib/btree/btreenode.cpp index 58992c12def..85299091441 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenode.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreenode.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreenode.h" #include "btreenode.hpp" -namespace search { -namespace btree { +namespace search::btree { BTreeNoLeafData BTreeNoLeafData::_instance; @@ -34,4 +32,3 @@ template class BTreeLeafNodeTemp<uint32_t, int32_t, MinMaxAggregated, template class BTreeLeafNodeTemp<uint32_t, BTreeNoLeafData, NoAggregated, 16>; } // namespace search::btree -} // namespace search diff --git a/searchlib/src/vespa/searchlib/btree/btreenode.h b/searchlib/src/vespa/searchlib/btree/btreenode.h index 60e4fde9119..695392ce091 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenode.h +++ b/searchlib/src/vespa/searchlib/btree/btreenode.h @@ -134,7 +134,7 @@ public: (void) data; } - static bool hasData(void) { return false; } + static bool hasData() { return false; } }; @@ -159,7 +159,7 @@ public: {} void setData(const DataT &data) { _data = data; } - const DataT &getData(void) const { return _data; } + const DataT &getData() const { return _data; } /** * This operator only works when using direct keys. References to @@ -189,7 +189,7 @@ public: } void setData(const BTreeNoLeafData &data) { (void) data; } - const BTreeNoLeafData &getData(void) const { return BTreeNoLeafData::_instance; } + const BTreeNoLeafData &getData() const { return BTreeNoLeafData::_instance; } /** * This operator only works when using direct keys. References to diff --git a/searchlib/src/vespa/searchlib/btree/btreenode.hpp b/searchlib/src/vespa/searchlib/btree/btreenode.hpp index 49e9fa373e2..5b423234b83 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenode.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreenode.hpp @@ -229,7 +229,7 @@ BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::cleanRange(uint32_t from, template <typename KeyT, typename DataT, typename AggrT, uint32_t NumSlots> void -BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::clean(void) +BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::clean() { if (validSlots() == 0) return; @@ -240,7 +240,7 @@ BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::clean(void) template <typename KeyT, typename DataT, typename AggrT, uint32_t NumSlots> void -BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::cleanFrozen(void) +BTreeNodeTT<KeyT, DataT, AggrT, NumSlots>::cleanFrozen() { assert(validSlots() <= NodeType::maxSlots()); assert(getFrozen()); @@ -368,7 +368,7 @@ stealSomeFromRightNode(BTreeInternalNode *victim, NodeAllocatorType &allocator) template <typename KeyT, typename AggrT, uint32_t NumSlots> void -BTreeInternalNode<KeyT, AggrT, NumSlots>::clean(void) +BTreeInternalNode<KeyT, AggrT, NumSlots>::clean() { ParentType::clean(); _validLeaves = 0; @@ -377,7 +377,7 @@ BTreeInternalNode<KeyT, AggrT, NumSlots>::clean(void) template <typename KeyT, typename AggrT, uint32_t NumSlots> void -BTreeInternalNode<KeyT, AggrT, NumSlots>::cleanFrozen(void) +BTreeInternalNode<KeyT, AggrT, NumSlots>::cleanFrozen() { ParentType::cleanFrozen(); _validLeaves = 0; diff --git a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.h b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.h index 4a6b9b1d873..a8db5a91c43 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.h +++ b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.h @@ -61,9 +61,9 @@ private: RefVector _leafHoldUntilFreeze; public: - BTreeNodeAllocator(void); + BTreeNodeAllocator(); - ~BTreeNodeAllocator(void); + ~BTreeNodeAllocator(); void disableFreeLists() { @@ -86,7 +86,7 @@ public: * Allocate leaf node. */ LeafNodeTypeRefPair - allocLeafNode(void); + allocLeafNode(); InternalNodeTypeRefPair thawNode(BTreeNode::Ref nodeRef, InternalNodeType *node); @@ -120,7 +120,7 @@ public: * Freeze all nodes that are not already frozen. */ void - freeze(void); + freeze(); /** * Try to free held nodes if nobody can be referencing them. @@ -138,7 +138,7 @@ public: transferHoldLists(generation_t generation); void - clearHoldLists(void); + clearHoldLists(); static bool isValidRef(BTreeNode::Ref ref) @@ -226,7 +226,7 @@ public: } std::vector<uint32_t> - startCompact(void) + startCompact() { return _nodeStore.startCompact(); } diff --git a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp index 0df47e1d186..e5736c387d1 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreenodeallocator.hpp @@ -14,7 +14,7 @@ namespace btree { template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -BTreeNodeAllocator(void) +BTreeNodeAllocator() : _nodeStore(), _internalToFreeze(), _leafToFreeze(), @@ -28,7 +28,7 @@ BTreeNodeAllocator(void) template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -~BTreeNodeAllocator(void) +~BTreeNodeAllocator() { assert(_internalToFreeze.empty()); assert(_leafToFreeze.empty()); @@ -70,7 +70,7 @@ template <typename KeyT, typename DataT, typename AggrT, typename BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: LeafNodeTypeRefPair BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -allocLeafNode(void) +allocLeafNode() { if (_leafHoldUntilFreeze.empty()) { LeafNodeTypeRefPair nodeRef = _nodeStore.allocLeafNode(); @@ -191,7 +191,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> void BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -freeze(void) +freeze() { // Freeze nodes. @@ -286,7 +286,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> void BTreeNodeAllocator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -clearHoldLists(void) +clearHoldLists() { _nodeStore.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/btree/btreenodestore.cpp b/searchlib/src/vespa/searchlib/btree/btreenodestore.cpp index d7f15cf9ca5..26862370fa6 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodestore.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreenodestore.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreenodestore.hpp" #include "btreenode.h" #include "btreerootbase.h" @@ -8,8 +7,7 @@ #include "btreenodeallocator.h" #include <vespa/searchlib/datastore/datastore.h> -namespace search { -namespace btree { +namespace search::btree { template class BTreeNodeStore<uint32_t, uint32_t, NoAggregated, @@ -35,6 +33,4 @@ typedef BTreeNodeStore<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS> MyNodeStore3; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreenodestore.h b/searchlib/src/vespa/searchlib/btree/btreenodestore.h index c4fed093802..61ef28a476d 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodestore.h +++ b/searchlib/src/vespa/searchlib/btree/btreenodestore.h @@ -76,9 +76,9 @@ private: BTreeNodeBufferType<LeafNodeType> _leafNodeType; public: - BTreeNodeStore(void); + BTreeNodeStore(); - ~BTreeNodeStore(void); + ~BTreeNodeStore(); void disableFreeLists() { @@ -207,7 +207,7 @@ public: } std::vector<uint32_t> - startCompact(void); + startCompact(); void finishCompact(const std::vector<uint32_t> &toHold); @@ -231,7 +231,7 @@ public: } void - clearHoldLists(void) + clearHoldLists() { _store.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/btree/btreenodestore.hpp b/searchlib/src/vespa/searchlib/btree/btreenodestore.hpp index 3a19203631d..f1fb84902f0 100644 --- a/searchlib/src/vespa/searchlib/btree/btreenodestore.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreenodestore.hpp @@ -44,7 +44,7 @@ BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -BTreeNodeStore(void) +BTreeNodeStore() : _store(), _internalNodeType(MIN_CLUSTERS, RefType::offsetSize()), _leafNodeType(MIN_CLUSTERS, RefType::offsetSize()) @@ -58,7 +58,7 @@ BTreeNodeStore(void) template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -~BTreeNodeStore(void) +~BTreeNodeStore() { _store.dropBuffers(); // Drop buffers before type handlers are dropped } @@ -68,7 +68,7 @@ template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS> std::vector<uint32_t> BTreeNodeStore<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS>:: -startCompact(void) +startCompact() { std::vector<uint32_t> iToHold = _store.startCompact(NODETYPE_INTERNAL); diff --git a/searchlib/src/vespa/searchlib/btree/btreeremover.cpp b/searchlib/src/vespa/searchlib/btree/btreeremover.cpp index 0c3bdd346b6..d7324ec4af9 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeremover.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreeremover.cpp @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreeremover.h" #include "btreenodeallocator.h" #include "btreerootbase.hpp" #include "btreeremover.hpp" #include "btreenode.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeRemover<uint32_t, uint32_t, NoAggregated>; template class BTreeRemover<uint32_t, BTreeNoLeafData, NoAggregated>; @@ -20,6 +15,4 @@ template class BTreeRemover<uint32_t, int32_t, MinMaxAggregated, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreeroot.cpp b/searchlib/src/vespa/searchlib/btree/btreeroot.cpp index 9fce40628d4..14db139486d 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeroot.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreeroot.cpp @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreeroot.h" #include "btreenodeallocator.h" #include "btreeiterator.hpp" #include "btreeroot.hpp" #include "btreenode.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeRootT<uint32_t, uint32_t, NoAggregated>; template class BTreeRootT<uint32_t, BTreeNoLeafData, NoAggregated>; @@ -22,6 +17,4 @@ template class BTreeRoot<uint32_t, int32_t, MinMaxAggregated, std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreeroot.h b/searchlib/src/vespa/searchlib/btree/btreeroot.h index b06050904f1..0c4871181f2 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeroot.h +++ b/searchlib/src/vespa/searchlib/btree/btreeroot.h @@ -84,16 +84,16 @@ public: } BTreeNode::Ref - getRoot(void) const + getRoot() const { return _frozenRoot; } size_t - size(void) const; + size() const; const NodeAllocatorType & - getAllocator(void) const + getAllocator() const { return _allocator; } diff --git a/searchlib/src/vespa/searchlib/btree/btreeroot.hpp b/searchlib/src/vespa/searchlib/btree/btreeroot.hpp index 6b39e142d28..1d38ab9fd3f 100644 --- a/searchlib/src/vespa/searchlib/btree/btreeroot.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreeroot.hpp @@ -211,7 +211,7 @@ template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT> size_t BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: -FrozenView::size(void) const +FrozenView::size() const { if (NodeAllocatorType::isValidRef(_frozenRoot)) { return _allocator.validLeaves(_frozenRoot); diff --git a/searchlib/src/vespa/searchlib/btree/btreerootbase.h b/searchlib/src/vespa/searchlib/btree/btreerootbase.h index 8d16402a030..98963f2398b 100644 --- a/searchlib/src/vespa/searchlib/btree/btreerootbase.h +++ b/searchlib/src/vespa/searchlib/btree/btreerootbase.h @@ -34,13 +34,13 @@ protected: static_assert(sizeof(_root) == sizeof(_frozenRoot), "BTree root reference size mismatch"); - BTreeRootBase(void); + BTreeRootBase(); BTreeRootBase(const BTreeRootBase &rhs); BTreeRootBase &operator=(const BTreeRootBase &rhs); - ~BTreeRootBase(void); + ~BTreeRootBase(); public: void @@ -67,19 +67,19 @@ public: } BTreeNode::Ref - getRoot(void) const + getRoot() const { return _root; } BTreeNode::Ref - getFrozenRoot(void) const + getFrozenRoot() const { return BTreeNode::Ref(_frozenRoot.load(std::memory_order_acquire)); } BTreeNode::Ref - getFrozenRootRelaxed(void) const + getFrozenRootRelaxed() const { return BTreeNode::Ref(_frozenRoot.load(std::memory_order_relaxed)); } @@ -91,7 +91,7 @@ public: } void - recycle(void) + recycle() { _root = BTreeNode::Ref(); _frozenRoot = BTreeNode::Ref().ref(); diff --git a/searchlib/src/vespa/searchlib/btree/btreestore.cpp b/searchlib/src/vespa/searchlib/btree/btreestore.cpp index a5c05d9bf2a..7de106c5dc7 100644 --- a/searchlib/src/vespa/searchlib/btree/btreestore.cpp +++ b/searchlib/src/vespa/searchlib/btree/btreestore.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "btreestore.h" #include "btreestore.hpp" #include "btreeiterator.hpp" -namespace search -{ - -namespace btree -{ +namespace search::btree { template class BTreeStore<uint32_t, uint32_t, NoAggregated, @@ -27,6 +22,4 @@ template class BTreeStore<uint32_t, int32_t, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/btree/btreestore.h b/searchlib/src/vespa/searchlib/btree/btreestore.h index 6cf44129d4e..e8153f94363 100644 --- a/searchlib/src/vespa/searchlib/btree/btreestore.h +++ b/searchlib/src/vespa/searchlib/btree/btreestore.h @@ -103,7 +103,7 @@ public: BTreeStore(bool init); - ~BTreeStore(void); + ~BTreeStore(); const NodeAllocatorType &getAllocator() const { return _allocator; } @@ -153,7 +153,7 @@ public: allocKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize); std::vector<uint32_t> - startCompact(void); + startCompact(); void finishCompact(const std::vector<uint32_t> &toHold); @@ -354,7 +354,7 @@ public: } void - clearHoldLists(void) + clearHoldLists() { _allocator.clearHoldLists(); _store.clearHoldLists(); @@ -370,7 +370,7 @@ public: } void - clearBuilder(void) + clearBuilder() { _builder.clear(); } diff --git a/searchlib/src/vespa/searchlib/btree/btreestore.hpp b/searchlib/src/vespa/searchlib/btree/btreestore.hpp index 94d1d99849e..33aa347f51f 100644 --- a/searchlib/src/vespa/searchlib/btree/btreestore.hpp +++ b/searchlib/src/vespa/searchlib/btree/btreestore.hpp @@ -60,7 +60,7 @@ BTreeStore(bool init) template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> -BTreeStore<KeyT, DataT, AggrT, CompareT,TraitsT, AggrCalcT>::~BTreeStore(void) +BTreeStore<KeyT, DataT, AggrT, CompareT,TraitsT, AggrCalcT>::~BTreeStore() { _builder.clear(); _store.dropBuffers(); // Drop buffers before type handlers are dropped @@ -123,7 +123,7 @@ allocKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize) template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, typename AggrCalcT> std::vector<uint32_t> -BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::startCompact(void) +BTreeStore<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::startCompact() { std::vector<uint32_t> ret = _store.startCompact(clusterLimit); for (uint32_t clusterSize = 1; clusterSize <= clusterLimit; ++clusterSize) { diff --git a/searchlib/src/vespa/searchlib/btree/minmaxaggrcalc.h b/searchlib/src/vespa/searchlib/btree/minmaxaggrcalc.h index 09cd8ea7d23..00f6b7a323c 100644 --- a/searchlib/src/vespa/searchlib/btree/minmaxaggrcalc.h +++ b/searchlib/src/vespa/searchlib/btree/minmaxaggrcalc.h @@ -10,40 +10,12 @@ namespace btree class MinMaxAggrCalc { public: - MinMaxAggrCalc(void) - { - } - - static bool - hasAggregated(void) - { - return true; - } - - static int32_t - getVal(int32_t val) - { - return val; - } - - static void - add(MinMaxAggregated &a, int32_t val) - { - a.add(val); - } - - static void - add(MinMaxAggregated &a, const MinMaxAggregated &ca) - { - a.add(ca); - } - - static void - add(MinMaxAggregated &a, const MinMaxAggregated &oldca, - const MinMaxAggregated &ca) - { - a.add(oldca, ca); - } + MinMaxAggrCalc() { } + static bool hasAggregated() { return true; } + static int32_t getVal(int32_t val) { return val; } + static void add(MinMaxAggregated &a, int32_t val) { a.add(val); } + static void add(MinMaxAggregated &a, const MinMaxAggregated &ca) { a.add(ca); } + static void add(MinMaxAggregated &a, const MinMaxAggregated &oldca, const MinMaxAggregated &ca) { a.add(oldca, ca); } /* Returns true if recalculation is needed */ static bool diff --git a/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h b/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h index 90040b129e7..4a99daef79c 100644 --- a/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h +++ b/searchlib/src/vespa/searchlib/btree/minmaxaggregated.h @@ -14,7 +14,7 @@ class MinMaxAggregated int32_t _max; public: - MinMaxAggregated(void) + MinMaxAggregated() : _min(std::numeric_limits<int32_t>::max()), _max(std::numeric_limits<int32_t>::min()) { } @@ -24,8 +24,8 @@ public: _max(max) { } - int32_t getMin(void) const { return _min; } - int32_t getMax(void) const { return _max; } + int32_t getMin() const { return _min; } + int32_t getMax() const { return _max; } void add(int32_t val) diff --git a/searchlib/src/vespa/searchlib/btree/noaggrcalc.h b/searchlib/src/vespa/searchlib/btree/noaggrcalc.h index 57d7ccd2f45..1e4d8af8556 100644 --- a/searchlib/src/vespa/searchlib/btree/noaggrcalc.h +++ b/searchlib/src/vespa/searchlib/btree/noaggrcalc.h @@ -10,12 +10,12 @@ namespace btree class NoAggrCalc { public: - NoAggrCalc(void) + NoAggrCalc() { } static bool - hasAggregated(void) + hasAggregated() { return false; } diff --git a/searchlib/src/vespa/searchlib/btree/noaggregated.h b/searchlib/src/vespa/searchlib/btree/noaggregated.h index 9f710840d5a..f378b713d42 100644 --- a/searchlib/src/vespa/searchlib/btree/noaggregated.h +++ b/searchlib/src/vespa/searchlib/btree/noaggregated.h @@ -10,9 +10,7 @@ namespace btree class NoAggregated { public: - NoAggregated(void) - { - } + NoAggregated() { } }; diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index fd369581a80..c6a7aaa80ac 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -29,6 +29,7 @@ vespa_add_library(searchlib_common OBJECT sortdata.cpp sortresults.cpp sortspec.cpp + threaded_compactable_lid_space.cpp tunefileinfo.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/common/documentsummary.cpp b/searchlib/src/vespa/searchlib/common/documentsummary.cpp index 03faf45c1a9..9dc270918cf 100644 --- a/searchlib/src/vespa/searchlib/common/documentsummary.cpp +++ b/searchlib/src/vespa/searchlib/common/documentsummary.cpp @@ -2,13 +2,12 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "documentsummary.h" #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filekit.h> -#include <vespa/searchlib/common/documentsummary.h> #include <vespa/vespalib/util/error.h> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.documentsummary"); using vespalib::getLastErrorString; diff --git a/searchlib/src/vespa/searchlib/common/fileheadercontext.cpp b/searchlib/src/vespa/searchlib/common/fileheadercontext.cpp index c5d1399c47d..84c78fc0fd4 100644 --- a/searchlib/src/vespa/searchlib/common/fileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/common/fileheadercontext.cpp @@ -1,29 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.fileheadercontext"); #include "fileheadercontext.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/fastos/timestamp.h> -namespace search -{ - -namespace common -{ +namespace search::common { using vespalib::GenericHeader; -FileHeaderContext::FileHeaderContext(void) +FileHeaderContext::FileHeaderContext() { } - -FileHeaderContext::~FileHeaderContext(void) +FileHeaderContext::~FileHeaderContext() { } - void FileHeaderContext::addCreateAndFreezeTime(GenericHeader &header) { @@ -33,7 +25,6 @@ FileHeaderContext::addCreateAndFreezeTime(GenericHeader &header) header.putTag(Tag("freezeTime", 0)); } - void FileHeaderContext::setFreezeTime(GenericHeader &header) { @@ -45,7 +36,4 @@ FileHeaderContext::setFreezeTime(GenericHeader &header) } } - -} // namespace common - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/fileheadercontext.h b/searchlib/src/vespa/searchlib/common/fileheadercontext.h index cb0f31edc38..cf536043047 100644 --- a/searchlib/src/vespa/searchlib/common/fileheadercontext.h +++ b/searchlib/src/vespa/searchlib/common/fileheadercontext.h @@ -19,10 +19,10 @@ namespace common class FileHeaderContext { public: - FileHeaderContext(void); + FileHeaderContext(); virtual - ~FileHeaderContext(void); + ~FileHeaderContext(); virtual void addTags(vespalib::GenericHeader &header, diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp index 595c91f9637..4be46867f3a 100644 --- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".common.foregroundtaskexecutor"); - #include "foregroundtaskexecutor.h" #include <vespa/vespalib/util/threadstackexecutor.h> using vespalib::ThreadStackExecutor; -namespace search -{ +namespace search { ForegroundTaskExecutor::ForegroundTaskExecutor() : ForegroundTaskExecutor(1) diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 6e8948b38c1..a1be37aebee 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -2,12 +2,10 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include "growablebitvector.h" ///////////////////////////////// -namespace search -{ +namespace search { using vespalib::GenerationHeldBase; using vespalib::GenerationHolder; diff --git a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h index 65b04570dcb..583ea1f4b92 100644 --- a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h +++ b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h @@ -6,7 +6,7 @@ namespace search { namespace common { /** - * Interface for a component that has a lid space that can be compacted and shrinked. + * Interface for a component that has a lid space that can be compacted and shrunk. */ struct ICompactableLidSpace { virtual ~ICompactableLidSpace() {} @@ -18,10 +18,15 @@ struct ICompactableLidSpace { virtual void compactLidSpace(uint32_t wantedDocLidLimit) = 0; /** - * Returns whether this lid space can be shrinked down to the wanted doc id limit. + * Returns whether this lid space can be shrunk down to the wanted doc id limit. */ virtual bool canShrinkLidSpace() const = 0; + /* + * Returns how much memory (in bytes) that can be saved by shrinking lid space. + */ + virtual size_t getEstimatedShrinkLidSpaceGain() const = 0; + /** * Shrinks this lid space down to the wanted doc id limit (frees memory resources). */ diff --git a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h index 69bb76e76c5..d0b06c26811 100644 --- a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h +++ b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper.h @@ -2,6 +2,8 @@ #pragma once +#include <cstdint> + namespace document { class GlobalId; } namespace search { diff --git a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h index 5f57ada4486..455faf6d37b 100644 --- a/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h +++ b/searchlib/src/vespa/searchlib/common/i_gid_to_lid_mapper_factory.h @@ -2,6 +2,8 @@ #pragma once +#include <memory> + namespace search { class IGidToLidMapper; diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp index 4344895ad63..f92087f7352 100644 --- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp +++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp @@ -3,6 +3,7 @@ #include "indexmetainfo.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/guard.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".indexmetainfo"); diff --git a/searchlib/src/vespa/searchlib/common/location.cpp b/searchlib/src/vespa/searchlib/common/location.cpp index 5374870773e..24481c47528 100644 --- a/searchlib/src/vespa/searchlib/common/location.cpp +++ b/searchlib/src/vespa/searchlib/common/location.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Copyright (C) 1999-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/common/location.h> -namespace search { -namespace common { +#include "location.h" -Location::Location(void) : +namespace search::common { + +Location::Location() : _zBoundingBox(0,0,0,0), _x(0), _y(0), @@ -202,4 +201,3 @@ bool Location::parse(const vespalib::string &locStr) } } -} diff --git a/searchlib/src/vespa/searchlib/common/location.h b/searchlib/src/vespa/searchlib/common/location.h index 702c44a1c36..e32f96627ed 100644 --- a/searchlib/src/vespa/searchlib/common/location.h +++ b/searchlib/src/vespa/searchlib/common/location.h @@ -17,7 +17,7 @@ private: bool getDimensionality(const char **pp); public: - Location(void); + Location(); bool getRankOnDistance() const { return _rankOnDistance; } bool getPruneOnDistance() const { return _pruneOnDistance; } uint32_t getXAspect() const { return _xAspect; } diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index be5a9f5ef9d..531305789b3 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> -#include <vespa/searchlib/common/mapnames.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/common/sortdata.h> +#include "mapnames.h" +#include "packets.h" +#include "sortdata.h" #include <vespa/searchlib/util/rawbuf.h> +#include <vespa/document/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> @@ -17,8 +16,7 @@ using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; -namespace search { -namespace fs4transport { +namespace search::fs4transport { /** * Persistent packet streamer. @@ -1787,4 +1785,3 @@ FS4PacketFactory::CreateFS4Packet(uint32_t pcode) } } -} diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index 00ebc6a7673..e760edd2d8f 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -1,24 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once +#include "transport.h" +#include "hitrank.h" #include <vespa/fnet/context.h> #include <vespa/fnet/ipacketstreamer.h> #include <vespa/fnet/packet.h> #include <vespa/fnet/databuffer.h> -#include <vespa/vespalib/util/memory.h> #include <vespa/document/base/globalid.h> #include <vespa/document/util/compressionconfig.h> -#include <vespa/searchlib/common/transport.h> -#include <vespa/searchlib/common/hitrank.h> +#include <vespa/vespalib/util/memory.h> #include <vespa/fastos/timestamp.h> #include <vector> -namespace search { - -namespace fs4transport { +namespace search::fs4transport { using vespalib::string; @@ -536,5 +532,3 @@ public: //========================================================================== } -} - diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.h b/searchlib/src/vespa/searchlib/common/partialbitvector.h index 79080511074..0520f4ef889 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.h +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.h @@ -27,7 +27,7 @@ public: PartialBitVector(Index start, Index end); PartialBitVector(const BitVector & org, Index start, Index end); - virtual ~PartialBitVector(void); + virtual ~PartialBitVector(); private: vespalib::alloc::Alloc _alloc; diff --git a/searchlib/src/vespa/searchlib/common/rcuvector.h b/searchlib/src/vespa/searchlib/common/rcuvector.h index bde7fb2accb..aeaec9b1022 100644 --- a/searchlib/src/vespa/searchlib/common/rcuvector.h +++ b/searchlib/src/vespa/searchlib/common/rcuvector.h @@ -58,7 +58,7 @@ private: } void expand(size_t newCapacity); void expandAndInsert(const T & v); - virtual void onExpand(); + virtual void onReallocation(); public: using ValueType = T; @@ -133,7 +133,7 @@ private: generation_t _generation; GenerationHolder _genHolderStore; - void onExpand() override; + void onReallocation() override; public: RcuVector(); diff --git a/searchlib/src/vespa/searchlib/common/rcuvector.hpp b/searchlib/src/vespa/searchlib/common/rcuvector.hpp index c68016dea81..9424331d014 100644 --- a/searchlib/src/vespa/searchlib/common/rcuvector.hpp +++ b/searchlib/src/vespa/searchlib/common/rcuvector.hpp @@ -60,7 +60,7 @@ RcuVectorBase<T>::expand(size_t newCapacity) { size_t holdSize = tmpData->capacity() * sizeof(T); vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); _genHolder.hold(std::move(hold)); - onExpand(); + onReallocation(); } template <typename T> @@ -95,6 +95,7 @@ RcuVectorBase<T>::shrink(size_t newSize) size_t holdSize = tmpData->capacity() * sizeof(T); vespalib::GenerationHeldBase::UP hold(new RcuVectorHeld<Array>(holdSize, std::move(tmpData))); _genHolder.hold(std::move(hold)); + onReallocation(); } } @@ -144,11 +145,11 @@ RcuVectorBase<T>::getMemoryUsage() const template <typename T> void -RcuVectorBase<T>::onExpand() { } +RcuVectorBase<T>::onReallocation() { } template <typename T> void -RcuVector<T>::onExpand() { +RcuVector<T>::onReallocation() { _genHolderStore.transferHoldLists(_generation); } diff --git a/searchlib/src/vespa/searchlib/common/resultset.cpp b/searchlib/src/vespa/searchlib/common/resultset.cpp index 0544e54b59b..a7d9758f397 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.cpp +++ b/searchlib/src/vespa/searchlib/common/resultset.cpp @@ -2,9 +2,8 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/common/resultset.h> -#include <vespa/searchlib/common/bitvector.h> +#include "resultset.h" +#include "bitvector.h" using vespalib::alloc::Alloc; @@ -13,7 +12,7 @@ namespace search { //Above 32M we hand back to the OS directly. constexpr size_t MMAP_LIMIT = 0x2000000; -ResultSet::ResultSet(void) +ResultSet::ResultSet() : _elemsUsedInRankedHitsArray(0u), _rankedHitsArrayAllocElements(0u), _bitOverflow(), @@ -41,7 +40,7 @@ ResultSet::ResultSet(const ResultSet &other) } -ResultSet::~ResultSet(void) +ResultSet::~ResultSet() { } @@ -78,14 +77,14 @@ ResultSet::setBitOverflow(BitVector::UP newBitOverflow) // Find number of hits ////////////////////////////////////////////////////////////////////// unsigned int -ResultSet::getNumHits(void) const +ResultSet::getNumHits() const { return (_bitOverflow) ? _bitOverflow->countTrueBits() : _elemsUsedInRankedHitsArray; } void -ResultSet::mergeWithBitOverflow(void) +ResultSet::mergeWithBitOverflow() { if ( ! _bitOverflow) { return; diff --git a/searchlib/src/vespa/searchlib/common/serialnumfileheadercontext.cpp b/searchlib/src/vespa/searchlib/common/serialnumfileheadercontext.cpp index d9b575678d9..932e405f08c 100644 --- a/searchlib/src/vespa/searchlib/common/serialnumfileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/common/serialnumfileheadercontext.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "serialnumfileheadercontext.h" #include <vespa/vespalib/data/fileheader.h> -namespace search -{ - -namespace common -{ - +namespace search::common { SerialNumFileHeaderContext::SerialNumFileHeaderContext( const FileHeaderContext &parentFileHeaderContext, @@ -31,6 +26,4 @@ SerialNumFileHeaderContext::addTags(vespalib::GenericHeader &header, header.putTag(Tag("serialNum", _serialNum)); } -} // namespace common - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/sort.cpp b/searchlib/src/vespa/searchlib/common/sort.cpp index 9d43e98457d..3af18f1344a 100644 --- a/searchlib/src/vespa/searchlib/common/sort.cpp +++ b/searchlib/src/vespa/searchlib/common/sort.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/common/sort.h> +#include "sort.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/common/sort.h b/searchlib/src/vespa/searchlib/common/sort.h index 231865321c0..44d4bfda925 100644 --- a/searchlib/src/vespa/searchlib/common/sort.h +++ b/searchlib/src/vespa/searchlib/common/sort.h @@ -6,10 +6,9 @@ #include <functional> #include <limits> #include <algorithm> +#include <cstring> - -namespace search -{ +namespace search { bool radix_prepare(unsigned int n, unsigned int last[257], unsigned int ptr[256], unsigned int cnt[256]); diff --git a/searchlib/src/vespa/searchlib/common/sortdata.cpp b/searchlib/src/vespa/searchlib/common/sortdata.cpp index a0923611b4e..d0e59510c1e 100644 --- a/searchlib/src/vespa/searchlib/common/sortdata.cpp +++ b/searchlib/src/vespa/searchlib/common/sortdata.cpp @@ -1,16 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/common/sortdata.h> +#include "sortdata.h" +#include <cassert> +#include <cstring> namespace search { namespace common { uint32_t -SortData::GetSize(uint32_t hitcnt, - const uint32_t *sortIndex) +SortData::GetSize(uint32_t hitcnt, const uint32_t *sortIndex) { if (hitcnt == 0) return 0; @@ -20,11 +18,9 @@ SortData::GetSize(uint32_t hitcnt, bool -SortData::Equals(uint32_t hitcnt, - const uint32_t *sortIndex_1, - const char *sortData_1, - const uint32_t *sortIndex_2, - const char *sortData_2) +SortData::Equals(uint32_t hitcnt, + const uint32_t *sortIndex_1, const char *sortData_1, + const uint32_t *sortIndex_2, const char *sortData_2) { if (hitcnt == 0) return true; @@ -42,11 +38,9 @@ SortData::Equals(uint32_t hitcnt, void -SortData::Copy(uint32_t hitcnt, - uint32_t *sortIndex_dst, - char *sortData_dst, - const uint32_t *sortIndex_src, - const char *sortData_src) +SortData::Copy(uint32_t hitcnt, + uint32_t *sortIndex_dst, char *sortData_dst, + const uint32_t *sortIndex_src, const char *sortData_src) { if (hitcnt == 0) return; @@ -61,5 +55,22 @@ SortData::Copy(uint32_t hitcnt, sortIndex_dst[hitcnt] - sortIndex_dst[0]); } +bool +SortDataIterator::Before(SortDataIterator *other, bool beforeOnMatch) +{ + uint32_t tlen = GetLen(); + uint32_t olen = other->GetLen(); + uint32_t mlen = (tlen <= olen) ? tlen : olen; + + if (mlen == 0) + return (tlen != 0 || beforeOnMatch); + + int res = memcmp(GetBuf(), other->GetBuf(), mlen); + + if (res != 0) + return (res < 0); + return (tlen < olen || (tlen == olen && beforeOnMatch)); +} + } } diff --git a/searchlib/src/vespa/searchlib/common/sortdata.h b/searchlib/src/vespa/searchlib/common/sortdata.h index 33a47841c9c..d1faed2add4 100644 --- a/searchlib/src/vespa/searchlib/common/sortdata.h +++ b/searchlib/src/vespa/searchlib/common/sortdata.h @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once +#include <cstdint> -namespace search { -namespace common { +namespace search::common { class SortData { @@ -46,13 +44,13 @@ private: public: SortDataIterator() - : _ofs(NULL), _ofs_end(NULL), _data(NULL), - _buf(NULL), _len(0) {} + : _ofs(nullptr), _ofs_end(nullptr), _data(nullptr), + _buf(nullptr), _len(0) {} void Next() { if (_ofs >= _ofs_end) { - _buf = NULL; + _buf = nullptr; _len = 0; return; } @@ -69,30 +67,14 @@ public: _ofs = idx; _ofs_end = idx + cnt; _data = data; - _buf = NULL; + _buf = nullptr; _len = 0; Next(); } uint32_t GetLen() const { return _len; } const char *GetBuf() const { return _buf; } - bool Before(SortDataIterator *other, bool beforeOnMatch = false) - { - uint32_t tlen = GetLen(); - uint32_t olen = other->GetLen(); - uint32_t mlen = (tlen <= olen) ? tlen : olen; - - if (mlen == 0) - return (tlen != 0 || beforeOnMatch); - - int res = memcmp(GetBuf(), other->GetBuf(), mlen); - - if (res != 0) - return (res < 0); - return (tlen < olen || (tlen == olen && beforeOnMatch)); - } + bool Before(SortDataIterator *other, bool beforeOnMatch = false); }; } -} - diff --git a/searchlib/src/vespa/searchlib/common/sortresults.h b/searchlib/src/vespa/searchlib/common/sortresults.h index bff8dbf199b..4de93716f8c 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.h +++ b/searchlib/src/vespa/searchlib/common/sortresults.h @@ -35,7 +35,7 @@ struct FastS_IResultSorter { /** * Destructor. No cleanup needed for base class. */ - virtual ~FastS_IResultSorter(void) {} + virtual ~FastS_IResultSorter() {} /** * @return should bitvector hits also be sorted? diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp new file mode 100644 index 00000000000..7d4b17229ae --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp @@ -0,0 +1,54 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <cstddef> +#include <cstdint> +#include "threaded_compactable_lid_space.h" +#include "isequencedtaskexecutor.h" +#include <future> + +namespace search { +namespace common { + +ThreadedCompactableLidSpace::ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, ISequencedTaskExecutor &executor, uint32_t executorId) + : _target(target), + _executor(executor), + _executorId(executorId) +{ +} + +ThreadedCompactableLidSpace::~ThreadedCompactableLidSpace() +{ +} + +void +ThreadedCompactableLidSpace::compactLidSpace(uint32_t wantedDocLidLimit) +{ + std::promise<bool> promise; + std::future<bool> future = promise.get_future(); + _executor.executeLambda(_executorId, [this, wantedDocLidLimit, &promise]() { _target->compactLidSpace(wantedDocLidLimit); promise.set_value(true); }); + (void) future.get(); +} + +bool +ThreadedCompactableLidSpace::canShrinkLidSpace() const +{ + return _target->canShrinkLidSpace(); +} + +size_t +ThreadedCompactableLidSpace::getEstimatedShrinkLidSpaceGain() const +{ + return _target->getEstimatedShrinkLidSpaceGain(); +} + +void +ThreadedCompactableLidSpace::shrinkLidSpace() +{ + std::promise<bool> promise; + std::future<bool> future = promise.get_future(); + _executor.executeLambda(_executorId, [this, &promise]() { _target->shrinkLidSpace(); promise.set_value(true); }); + (void) future.get(); +} + +} +} diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h new file mode 100644 index 00000000000..e902397692b --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_compactable_lid_space.h" +#include <memory> + +namespace search { + +class ISequencedTaskExecutor; + +namespace common { + +/** + * Adapter class for a component that has a lid space that can be + * compacted and shrunk where the write operations are performed by a + * specific thread. + */ +class ThreadedCompactableLidSpace : public ICompactableLidSpace +{ + std::shared_ptr<ICompactableLidSpace> _target; + ISequencedTaskExecutor &_executor; + uint32_t _executorId; +public: + ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, ISequencedTaskExecutor &executor, uint32_t executorId); + virtual ~ThreadedCompactableLidSpace() override; + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; + virtual bool canShrinkLidSpace() const override; + virtual size_t getEstimatedShrinkLidSpaceGain() const override; + virtual void shrinkLidSpace() override; +}; + +} +} diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h index d41b636ec8d..b523fbaee99 100644 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ b/searchlib/src/vespa/searchlib/common/transport.h @@ -2,9 +2,9 @@ #pragma once -namespace search { +#include <cstdint> -namespace fs4transport { +namespace search::fs4transport { /** * Instead of using a 32-bit number to send the 'usehardware' flag, we @@ -281,6 +281,4 @@ enum packetcode { PCODE_LastCode = 223 // Used for consistency checking only, must be last. }; -} // namespace fs4transport -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp index 231407fadca..89ae6f42327 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tunefileinfo.h" - -namespace search -{ - +namespace search { } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.h b/searchlib/src/vespa/searchlib/common/tunefileinfo.h index 5986a8f0f18..5365357e099 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.h +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.h @@ -5,6 +5,7 @@ #include <sys/mman.h> #include <linux/mman.h> #include <fcntl.h> +#include <memory> namespace search { @@ -104,7 +105,7 @@ private: int _mmapFlags; int _advise; public: - TuneFileRandRead(void) + TuneFileRandRead() : _tuneControl(NORMAL), _mmapFlags(0), _advise(0) @@ -163,7 +164,7 @@ public: TuneFileSeqRead _read; TuneFileSeqWrite _write; - TuneFileIndexing(void) : _read(), _write() {} + TuneFileIndexing() : _read(), _write() {} TuneFileIndexing(const TuneFileSeqRead &r, const TuneFileSeqWrite &w) : _read(r), _write(w) { } @@ -222,7 +223,7 @@ class TuneFileAttributes public: TuneFileSeqWrite _write; - TuneFileAttributes(void) : _write() { } + TuneFileAttributes() : _write() { } bool operator==(const TuneFileAttributes &rhs) const { return _write == rhs._write; @@ -244,7 +245,7 @@ public: TuneFileSeqWrite _write; TuneFileRandRead _randRead; - TuneFileSummary(void) : _seqRead(), _write(), _randRead() { } + TuneFileSummary() : _seqRead(), _write(), _randRead() { } bool operator==(const TuneFileSummary &rhs) const { return _seqRead == rhs._seqRead && @@ -277,14 +278,14 @@ public: bool operator==(const TuneFileDocumentDB &rhs) const { return _index == rhs._index && _attr == rhs._attr && - _summary == rhs._summary; + _summary == rhs._summary; } bool operator!=(const TuneFileDocumentDB &rhs) const { return _index != rhs._index || _attr != rhs._attr || - _summary != rhs._summary; + _summary != rhs._summary; } }; -}
\ No newline at end of file +} diff --git a/searchlib/src/vespa/searchlib/datastore/array_store.h b/searchlib/src/vespa/searchlib/datastore/array_store.h index 038ccdd01e9..a459c6dac2c 100644 --- a/searchlib/src/vespa/searchlib/datastore/array_store.h +++ b/searchlib/src/vespa/searchlib/datastore/array_store.h @@ -97,7 +97,7 @@ public: // Pass on hold list management to underlying store void transferHoldLists(generation_t generation) { _store.transferHoldLists(generation); } void trimHoldLists(generation_t firstUsed) { _store.trimHoldLists(firstUsed); } - vespalib::GenerationHolder &getGenerationHolder(void) { return _store.getGenerationHolder(); } + vespalib::GenerationHolder &getGenerationHolder() { return _store.getGenerationHolder(); } void setInitializing(bool initializing) { _store.setInitializing(initializing); } // Should only be used for unit testing diff --git a/searchlib/src/vespa/searchlib/datastore/array_store_config.cpp b/searchlib/src/vespa/searchlib/datastore/array_store_config.cpp index 1710e41069b..46a55a83776 100644 --- a/searchlib/src/vespa/searchlib/datastore/array_store_config.cpp +++ b/searchlib/src/vespa/searchlib/datastore/array_store_config.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "array_store_config.h" +#include <cassert> -namespace search { -namespace datastore { +namespace search::datastore { ArrayStoreConfig::ArrayStoreConfig(size_t maxSmallArraySize, const AllocSpec &defaultSpec) : _allocSpecs() @@ -63,4 +62,3 @@ ArrayStoreConfig::optimizeForHugePage(size_t maxSmallArraySize, } } -} diff --git a/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp b/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp index 8d01bc09710..f725e90b295 100644 --- a/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/datastore/buffer_type.cpp @@ -45,7 +45,7 @@ BufferTypeBase::getReservedElements(uint32_t bufferId) const } void -BufferTypeBase::flushLastUsed(void) +BufferTypeBase::flushLastUsed() { if (_lastUsedElems != NULL) { _activeUsedElems += *_lastUsedElems; diff --git a/searchlib/src/vespa/searchlib/datastore/bufferstate.cpp b/searchlib/src/vespa/searchlib/datastore/bufferstate.cpp index ab45a5e7f3d..7c8483028de 100644 --- a/searchlib/src/vespa/searchlib/datastore/bufferstate.cpp +++ b/searchlib/src/vespa/searchlib/datastore/bufferstate.cpp @@ -8,13 +8,13 @@ using vespalib::alloc::Alloc; namespace search { namespace datastore { -BufferState::FreeListList::~FreeListList(void) +BufferState::FreeListList::~FreeListList() { assert(_head == NULL); // Owner should have disabled free lists } -BufferState::BufferState(void) +BufferState::BufferState() : _usedElems(0), _allocElems(0), _deadElems(0u), @@ -36,7 +36,7 @@ BufferState::BufferState(void) } -BufferState::~BufferState(void) +BufferState::~BufferState() { assert(_state == FREE); assert(_freeListList == NULL); @@ -86,7 +86,7 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, void -BufferState::onHold(void) +BufferState::onHold() { assert(_state == ACTIVE); assert(_typeHandler != NULL); @@ -180,7 +180,7 @@ BufferState::setFreeListList(FreeListList *freeListList) void -BufferState::addToFreeListList(void) +BufferState::addToFreeListList() { assert(_freeListList != NULL && _freeListList->_head != this); assert(_nextHasFree == NULL); @@ -199,7 +199,7 @@ BufferState::addToFreeListList(void) void -BufferState::removeFromFreeListList(void) +BufferState::removeFromFreeListList() { assert(_freeListList != NULL); assert(_nextHasFree != NULL); @@ -220,7 +220,7 @@ BufferState::removeFromFreeListList(void) void -BufferState::disableElemHoldList(void) +BufferState::disableElemHoldList() { _disableElemHoldList = true; } diff --git a/searchlib/src/vespa/searchlib/datastore/entryref.h b/searchlib/src/vespa/searchlib/datastore/entryref.h index 40b1053757f..660c4df8abf 100644 --- a/searchlib/src/vespa/searchlib/datastore/entryref.h +++ b/searchlib/src/vespa/searchlib/datastore/entryref.h @@ -11,10 +11,10 @@ class EntryRef { protected: uint32_t _ref; public: - EntryRef(void) : _ref(0u) { } + EntryRef() : _ref(0u) { } EntryRef(uint32_t ref_) : _ref(ref_) { } - uint32_t ref(void) const { return _ref; } - bool valid(void) const { return _ref != 0u; } + uint32_t ref() const { return _ref; } + bool valid() const { return _ref != 0u; } bool operator==(const EntryRef &rhs) const { return _ref == rhs._ref; } bool operator!=(const EntryRef &rhs) const { return _ref != rhs._ref; } bool operator <(const EntryRef &rhs) const { return _ref < rhs._ref; } diff --git a/searchlib/src/vespa/searchlib/datastore/unique_store.h b/searchlib/src/vespa/searchlib/datastore/unique_store.h index a82752977be..ec24d03b3e2 100644 --- a/searchlib/src/vespa/searchlib/datastore/unique_store.h +++ b/searchlib/src/vespa/searchlib/datastore/unique_store.h @@ -104,7 +104,7 @@ public: // Pass on hold list management to underlying store void transferHoldLists(generation_t generation); void trimHoldLists(generation_t firstUsed); - vespalib::GenerationHolder &getGenerationHolder(void) { return _store.getGenerationHolder(); } + vespalib::GenerationHolder &getGenerationHolder() { return _store.getGenerationHolder(); } void setInitializing(bool initializing) { _store.setInitializing(initializing); } void freeze(); uint32_t getNumUniques() const; diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index d9b7237c065..35e8ef6ca11 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_add_library(searchlib_diskindex OBJECT pagedict4file.cpp pagedict4randread.cpp wordnummapper.cpp + zcbuf.cpp zcposocc.cpp zcposocciterators.cpp zcposoccrandread.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index bd896be10d6..e7b868f127c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -2,21 +2,14 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.bitvectorfile"); -#include <vespa/searchlib/common/bitvector.h> #include "bitvectorfile.h" #include <vespa/searchlib/index/bitvectorkeys.h> +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/common/fileheadercontext.h> - -namespace search -{ -namespace diskindex -{ +namespace search::diskindex { using vespalib::nbostream; using search::index::BitVectorWordSingleKey; @@ -46,7 +39,7 @@ BitVectorFileWrite::BitVectorFileWrite(BitVectorKeyScope scope) } -BitVectorFileWrite::~BitVectorFileWrite(void) +BitVectorFileWrite::~BitVectorFileWrite() { // No implicit close() call, but cleanup memory allocations. delete _datFile; @@ -167,7 +160,7 @@ BitVectorFileWrite::addWordSingle(uint64_t wordNum, void -BitVectorFileWrite::flush(void) +BitVectorFileWrite::flush() { Parent::flush(); _datFile->Flush(); @@ -175,7 +168,7 @@ BitVectorFileWrite::flush(void) void -BitVectorFileWrite::sync(void) +BitVectorFileWrite::sync() { flush(); Parent::syncCommon(); @@ -184,7 +177,7 @@ BitVectorFileWrite::sync(void) void -BitVectorFileWrite::close(void) +BitVectorFileWrite::close() { size_t bitmapbytes = BitVector::getFileBytes(_docIdLimit); @@ -236,7 +229,4 @@ BitVectorCandidate::checkPointRead(nbostream &in) } } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 9a88f4bea21..1d734b61f3e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -36,7 +36,7 @@ private: public: BitVectorFileWrite(BitVectorKeyScope scope); - ~BitVectorFileWrite(void); + ~BitVectorFileWrite(); /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -63,13 +63,13 @@ public: addWordSingle(uint64_t wordNum, const BitVector &bitVector); void - flush(void); + flush(); void - sync(void); + sync(); void - close(void); + close(); void makeDatHeader(const common::FileHeaderContext &fileHeaderContext); @@ -113,7 +113,7 @@ public: ~BitVectorCandidate(); void - clear(void) + clear() { if (__builtin_expect(_numDocs > _bitVectorLimit, false)) { _bv->clear(); @@ -156,13 +156,13 @@ public: * Get number of documents buffered. This might include duplicates. */ uint64_t - getNumDocs(void) const + getNumDocs() const { return _numDocs; } bool - empty(void) const + empty() const { return _numDocs == 0; } @@ -172,13 +172,13 @@ public: * populated. */ bool - getCrossedBitVectorLimit(void) const + getCrossedBitVectorLimit() const { return _numDocs > _bitVectorLimit; } BitVector & - getBitVector(void) + getBitVector() { return *_bv; } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 82c46d2172f..a779f3e7a6a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -2,21 +2,14 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.bitvectoridxfile"); -#include <vespa/searchlib/common/bitvector.h> #include "bitvectoridxfile.h" #include <vespa/searchlib/index/bitvectorkeys.h> +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/common/fileheadercontext.h> - -namespace search -{ -namespace diskindex -{ +namespace search::diskindex { using vespalib::nbostream; using search::index::BitVectorWordSingleKey; @@ -48,7 +41,7 @@ BitVectorIdxFileWrite::BitVectorIdxFileWrite(BitVectorKeyScope scope) } -BitVectorIdxFileWrite::~BitVectorIdxFileWrite(void) +BitVectorIdxFileWrite::~BitVectorIdxFileWrite() { // No implicit close() call, but cleanup memory allocations. delete _idxFile; @@ -56,7 +49,7 @@ BitVectorIdxFileWrite::~BitVectorIdxFileWrite(void) uint64_t -BitVectorIdxFileWrite::idxSize(void) const +BitVectorIdxFileWrite::idxSize() const { return _idxHeaderLen + static_cast<int64_t>(_numKeys) * sizeof(BitVectorWordSingleKey); @@ -186,7 +179,7 @@ BitVectorIdxFileWrite::addWordSingle(uint64_t wordNum, uint32_t numDocs) void -BitVectorIdxFileWrite::flush(void) +BitVectorIdxFileWrite::flush() { _idxFile->Flush(); @@ -204,7 +197,7 @@ BitVectorIdxFileWrite::syncCommon() void -BitVectorIdxFileWrite::sync(void) +BitVectorIdxFileWrite::sync() { flush(); syncCommon(); @@ -212,7 +205,7 @@ BitVectorIdxFileWrite::sync(void) void -BitVectorIdxFileWrite::close(void) +BitVectorIdxFileWrite::close() { if (_idxFile != NULL) { if (_idxFile->IsOpened()) { @@ -227,7 +220,4 @@ BitVectorIdxFileWrite::close(void) } } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index 269b6e659af..fda167241d3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -49,18 +49,14 @@ protected: uint32_t _idxHeaderLen; BitVectorKeyScope _scope; - uint64_t - idxSize(void) const; - - void - checkPointWriteCommon(vespalib::nbostream &out); - + uint64_t idxSize() const; + void checkPointWriteCommon(vespalib::nbostream &out); void syncCommon(); public: BitVectorIdxFileWrite(BitVectorKeyScope scope); - ~BitVectorIdxFileWrite(void); + ~BitVectorIdxFileWrite(); /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -88,13 +84,13 @@ public: addWordSingle(uint64_t wordNum, uint32_t numDocs); void - flush(void); + flush(); void - sync(void); + sync(); void - close(void); + close(); static uint32_t getBitVectorLimit(uint32_t docIdLimit) diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp index 344adae485c..fa3411b7835 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.cpp @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bitvectorkeyscope.h" #include <vespa/vespalib/objects/nbostream.h> - +#include <cassert> using search::diskindex::BitVectorKeyScope; -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { const char *getBitVectorKeyScopeSuffix(BitVectorKeyScope scope) { @@ -25,8 +20,6 @@ const char *getBitVectorKeyScopeSuffix(BitVectorKeyScope scope) } -} - namespace { @@ -48,8 +41,7 @@ const BitVectorKeyScope scopes[] = { BitVectorKeyScope::SHARED_WORDS, } -namespace vespalib -{ +namespace vespalib { nbostream & operator<<(nbostream &stream, const BitVectorKeyScope &scope) diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp index d0e611e3136..e990b57a601 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp @@ -1,23 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "dictionarywordreader.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/index/olddictionaryfile.h> #include <vespa/vespalib/util/error.h> + +#include <vespa/log/log.h> LOG_SETUP(".diskindex.dictionarywordreader"); -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { using vespalib::getLastErrorString; using index::SchemaUtil; -DictionaryWordReader::DictionaryWordReader(void) +DictionaryWordReader::DictionaryWordReader() : _word(), _wordNum(noWordNumHigh()), _old2newwordfile(), @@ -26,7 +22,7 @@ DictionaryWordReader::DictionaryWordReader(void) } -DictionaryWordReader::~DictionaryWordReader(void) +DictionaryWordReader::~DictionaryWordReader() { } @@ -56,7 +52,7 @@ DictionaryWordReader::open(const vespalib::stringref &dictionaryName, } void -DictionaryWordReader::close(void) +DictionaryWordReader::close() { if (!_dictFile->close()) LOG(error, "Error closing input dictionary"); @@ -65,7 +61,4 @@ DictionaryWordReader::close(void) _old2newwordfile->Close(); } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h index 744b73bf370..f47c067df0e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h @@ -64,27 +64,27 @@ private: std::unique_ptr<DictionaryFileSeqRead> _dictFile; void - allocFiles(void); + allocFiles(); static uint64_t - noWordNumHigh(void) + noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } static uint64_t - noWordNum(void) + noWordNum() { return 0u; } public: - DictionaryWordReader(void); + DictionaryWordReader(); - ~DictionaryWordReader(void); + ~DictionaryWordReader(); bool - isValid(void) const + isValid() const { return _wordNum != noWordNumHigh(); } @@ -100,7 +100,7 @@ public: } void - read(void) + read() { _dictFile->readWord(_word, _wordNum, _counts); } @@ -111,7 +111,7 @@ public: const TuneFileSeqRead &tuneFileRead); void - close(void); + close(); void writeNewWordNum(uint64_t newWordNum) diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp index b66f61095dd..5e39197c07c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp @@ -61,7 +61,7 @@ DiskIndex::DiskIndex(const vespalib::string &indexDir, size_t cacheSize) DiskIndex::~DiskIndex() {} bool -DiskIndex::loadSchema(void) +DiskIndex::loadSchema() { vespalib::string schemaName = _indexDir + "/schema.txt"; if (!_schema.loadFromFile(schemaName)) { diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index 10849b4b70a..14b843b33f0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.disktermblueprint"); - #include "disktermblueprint.h" #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> @@ -11,6 +7,9 @@ LOG_SETUP(".diskindex.disktermblueprint"); #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> +LOG_SETUP(".diskindex.disktermblueprint"); + using search::BitVectorIterator; using search::fef::TermFieldMatchDataArray; using search::index::Schema; diff --git a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp index 073c4b79031..4d4d2e5ce03 100644 --- a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp @@ -1,24 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.docidmapper"); +#include "docidmapper.h" #include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/fastlib/io/bufferedfile.h> -#include "docidmapper.h" #define NO_DOC static_cast<uint32_t>(-1) -namespace search -{ - -namespace diskindex -{ - +namespace search::diskindex { -DocIdMapping::DocIdMapping(void) +DocIdMapping::DocIdMapping() : _docIdLimit(0u), _selector(NULL), _selectorId(0) @@ -27,7 +19,7 @@ DocIdMapping::DocIdMapping(void) void -DocIdMapping::clear(void) +DocIdMapping::clear() { _docIdLimit = 0; _selector = NULL; @@ -68,6 +60,4 @@ DocIdMapping::readDocIdLimit(const vespalib::string &mergedDir) -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp index 9eaac550192..219a8929382 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp @@ -2,18 +2,17 @@ // Copyright (C) 2002-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "extposocc.h" -#include <vespa/vespalib/objects/nbostream.h> +#include "zcposocc.h" +#include "fileheader.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> -#include "zcposocc.h" -#include "fileheader.h" +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/log/log.h> LOG_SETUP(".diskindex.extposocc"); using search::index::PostingListFileSeqRead; diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index 279a73935f5..39d95f66a7a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -1,19 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/objects/nbostream.h> #include "fieldreader.h" #include "zcposocc.h" #include "extposocc.h" -#include <vespa/vespalib/util/error.h> #include "pagedict4file.h" +#include <vespa/vespalib/util/error.h> +#include <vespa/vespalib/objects/nbostream.h> + +#include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldreader"); #define NO_DOC static_cast<uint32_t>(-1) -namespace -{ +namespace { vespalib::string PosOccIdCooked = "PosOcc.3.Cooked"; @@ -25,14 +24,9 @@ using search::index::SchemaUtil; using search::bitcompression::PosOccFieldParams; using search::bitcompression::PosOccFieldsParams; -namespace search -{ - -namespace diskindex -{ - +namespace search::diskindex { -FieldReader::FieldReader(void) +FieldReader::FieldReader() : _wordNum(noWordNumHigh()), _docIdAndFeatures(), _dictFile(), @@ -48,13 +42,13 @@ FieldReader::FieldReader(void) } -FieldReader::~FieldReader(void) +FieldReader::~FieldReader() { } void -FieldReader::readCounts(void) +FieldReader::readCounts() { PostingListCounts counts; _dictFile->readWord(_word, _oldWordNum, counts); @@ -70,7 +64,7 @@ FieldReader::readCounts(void) void -FieldReader::readDocIdAndFeatures(void) +FieldReader::readDocIdAndFeatures() { _oldposoccfile->readDocIdAndFeatures(_docIdAndFeatures); _docIdAndFeatures._docId = _docIdMapper.mapDocId(_docIdAndFeatures._docId); @@ -78,7 +72,7 @@ FieldReader::readDocIdAndFeatures(void) void -FieldReader::read(void) +FieldReader::read() { for (;;) { while (_residue == 0) { @@ -98,7 +92,7 @@ FieldReader::read(void) bool -FieldReader::allowRawFeatures(void) +FieldReader::allowRawFeatures() { return true; } @@ -182,7 +176,7 @@ FieldReader::open(const vespalib::string &prefix, bool -FieldReader::close(void) +FieldReader::close() { bool ret = true; @@ -320,14 +314,14 @@ FieldReaderStripInfo::FieldReaderStripInfo(const IndexIterator &index) bool -FieldReaderStripInfo::allowRawFeatures(void) +FieldReaderStripInfo::allowRawFeatures() { return false; } void -FieldReaderStripInfo::read(void) +FieldReaderStripInfo::read() { typedef search::index::WordDocElementFeatures Element; @@ -379,7 +373,4 @@ FieldReaderStripInfo::getFeatureParams(PostingListParams ¶ms) params.erase("encoding"); } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index b3cf6446419..ddae3837930 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -60,34 +60,34 @@ protected: vespalib::string _word; static uint64_t - noWordNumHigh(void) + noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } static uint64_t - noWordNum(void) + noWordNum() { return 0u; } void - readCounts(void); + readCounts(); void - readDocIdAndFeatures(void); + readDocIdAndFeatures(); public: - FieldReader(void); + FieldReader(); virtual - ~FieldReader(void); + ~FieldReader(); virtual void - read(void); + read(); virtual bool - allowRawFeatures(void); + allowRawFeatures(); void write(FieldWriter &writer) @@ -99,7 +99,7 @@ public: } bool - isValid(void) const + isValid() const { return _wordNum != noWordNumHigh(); } @@ -128,7 +128,7 @@ public: open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead); virtual bool - close(void); + close(); /* * To be called between words, not in the middle of one. @@ -149,7 +149,7 @@ public: getFeatureParams(PostingListParams ¶ms); uint32_t - getDocIdLimit(void) const + getDocIdLimit() const { return _docIdLimit; } @@ -200,10 +200,10 @@ public: FieldReaderStripInfo(const IndexIterator &index); virtual bool - allowRawFeatures(void) override; + allowRawFeatures() override; virtual void - read(void) override; + read() override; virtual void getFeatureParams(PostingListParams ¶ms) override; diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index ac1f80be6af..306d4d0b9b6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -1,20 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/objects/nbostream.h> #include "fieldwriter.h" #include "zcposocc.h" #include "extposocc.h" -#include <vespa/vespalib/util/error.h> #include "pagedict4file.h" -LOG_SETUP(".diskindex.fieldwriter"); +#include <vespa/vespalib/util/error.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/log/log.h> -namespace search -{ +LOG_SETUP(".diskindex.fieldwriter"); -namespace diskindex -{ +namespace search::diskindex { using vespalib::nbostream; using vespalib::getLastErrorString; @@ -113,7 +109,7 @@ FieldWriter::lateOpen(const TuneFileSeqWrite &tuneFileWrite, void -FieldWriter::flush(void) +FieldWriter::flush() { _posoccfile->flushWord(); PostingListCounts &counts = _posoccfile->getCounts(); @@ -155,7 +151,7 @@ FieldWriter::newWord(const vespalib::stringref &word) bool -FieldWriter::close(void) +FieldWriter::close() { bool ret = true; flush(); @@ -253,7 +249,4 @@ FieldWriter::remove(const vespalib::string &prefix) } } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 46d5ce85383..0b51ae6cea9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -32,7 +32,7 @@ private: uint64_t _wordNum; uint32_t _prevDocId; - static uint64_t noWordNum(void) { return 0u; } + static uint64_t noWordNum() { return 0u; } public: using DictionaryFileSeqWrite = index::DictionaryFileSeqWrite; diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index 5c67ea5f064..6524710a35d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -1,23 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "fileheader.h" #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/fastos/file.h> +#include <vespa/log/log.h> LOG_SETUP(".diskindex.fileheader"); -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { using bitcompression::FeatureDecodeContextBE; -FileHeader::FileHeader(void) +FileHeader::FileHeader() : _bigEndian(false), _hostEndian(false), _completed(false), @@ -30,7 +26,7 @@ FileHeader::FileHeader(void) } -FileHeader::~FileHeader(void) +FileHeader::~FileHeader() { } @@ -158,8 +154,4 @@ FileHeader::taste(const vespalib::string &name, return taste(name, tuneFileRead); } - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.h b/searchlib/src/vespa/searchlib/diskindex/fileheader.h index 05db2d22e10..3085c3d18a6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.h @@ -25,9 +25,9 @@ private: std::vector<vespalib::string> _formats; public: - FileHeader(void); + FileHeader(); - ~FileHeader(void); + ~FileHeader(); bool taste(const vespalib::string &name, @@ -42,31 +42,31 @@ public: const TuneFileRandRead &tuneFileSearch); bool - getBigEndian(void) const + getBigEndian() const { return _bigEndian; } bool - getHostEndian(void) const + getHostEndian() const { return _hostEndian; } uint32_t - getVersion(void) const + getVersion() const { return _version; } uint32_t - getHeaderLen(void) const + getHeaderLen() const { return _headerLen; } const std::vector<vespalib::string> & - getFormats(void) const + getFormats() const { return _formats; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 8cae80fb892..ea66a3435d1 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -3,18 +3,14 @@ #include "fusion.h" #include "fieldreader.h" -#include "fieldwriter.h" #include "dictionarywordreader.h" #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/fslimits.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/common/documentsummary.h> -#include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/searchlib/index/postinglistparams.h> -#include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/vespalib/util/error.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".diskindex.fusion"); @@ -446,7 +442,7 @@ Fusion::makeTmpDirs() } bool -Fusion::CleanTmpDirs(void) +Fusion::CleanTmpDirs() { uint32_t i = 0; for (;;) { @@ -483,14 +479,14 @@ Fusion::CleanTmpDirs(void) bool -Fusion::checkSchemaCompat(void) +Fusion::checkSchemaCompat() { return true; } bool -Fusion::readSchemaFiles(void) +Fusion::readSchemaFiles() { OldIndexIterator oldIndexIt = _oldIndexes.begin(); OldIndexIterator oldIndexIte = _oldIndexes.end(); diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index 1cc23c61f10..2e01425f519 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -45,7 +45,7 @@ private: index::Schema::SP _schema; public: - FusionInputIndex(void) + FusionInputIndex() : _path(), _wordNumMapping(), _docIdMapping(), @@ -55,7 +55,7 @@ public: } virtual - ~FusionInputIndex(void) + ~FusionInputIndex() { } @@ -66,7 +66,7 @@ public: } const vespalib::string & - getPath(void) const + getPath() const { return _path; } @@ -78,37 +78,37 @@ public: } const vespalib::string & - getTmpPath(void) const + getTmpPath() const { return _tmpPath; } const WordNumMapping & - getWordNumMapping(void) const + getWordNumMapping() const { return _wordNumMapping; } WordNumMapping & - getWordNumMapping(void) + getWordNumMapping() { return _wordNumMapping; } const DocIdMapping & - getDocIdMapping(void) const + getDocIdMapping() const { return _docIdMapping; } DocIdMapping & - getDocIdMapping(void) + getDocIdMapping() { return _docIdMapping; } const index::Schema & - getSchema(void) const + getSchema() const { assert(_schema.get() != NULL); return *_schema.get(); @@ -135,7 +135,7 @@ public: const search::common::FileHeaderContext &fileHeaderContext); virtual - ~Fusion(void); + ~Fusion(); void SetOldIndexList(const std::vector<vespalib::string> &oldIndexList); @@ -166,13 +166,13 @@ public: void makeTmpDirs(); - bool CleanTmpDirs(void); + bool CleanTmpDirs(); bool - readSchemaFiles(void); + readSchemaFiles(); bool - checkSchemaCompat(void); + checkSchemaCompat(); template <class Reader, class Writer> static bool @@ -182,7 +182,7 @@ protected: bool ReadMappingFiles(const SchemaUtil::IndexIterator *index); bool ReleaseMappingTables(); - static unsigned int noGen(void) + static unsigned int noGen() { return static_cast<unsigned int>(-1); } @@ -215,7 +215,7 @@ protected: const search::common::FileHeaderContext &_fileHeaderContext; const Schema & - getSchema(void) const + getSchema() const { assert(_schema != NULL); return *_schema; @@ -235,13 +235,13 @@ public: } std::vector<std::shared_ptr<OldIndex> > & - getOldIndexes(void) + getOldIndexes() { return _oldIndexes; } virtual OldIndex * - allocOldIndex(void) + allocOldIndex() { return new OldIndex; } diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index 20d8a045efb..70d2caa1659 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -29,7 +29,7 @@ using index::schema::DataType; using vespalib::getLastErrorString; static uint32_t -noWordPos(void) +noWordPos() { return std::numeric_limits<uint32_t>::max(); } @@ -41,9 +41,9 @@ public: FieldWriter *_fieldWriter; DocIdAndFeatures _docIdAndFeatures; - FileHandle(void); + FileHandle(); - ~FileHandle(void); + ~FileHandle(); void open(const vespalib::stringref &dir, @@ -53,7 +53,7 @@ public: const FileHeaderContext &fileHeaderContext); void - close(void); + close(); }; @@ -73,16 +73,16 @@ public: uint32_t fieldId, IndexBuilder *ib); - ~FieldHandle(void); + ~FieldHandle(); static uint32_t - noDocRef(void) + noDocRef() { return std::numeric_limits<uint32_t>::max(); } static uint32_t - noElRef(void) + noElRef() { return std::numeric_limits<uint32_t>::max(); } @@ -100,19 +100,19 @@ public: } uint32_t - getDocId(void) const + getDocId() const { return _docId; } uint32_t - getNumElements(void) const + getNumElements() const { return _numElements; } void - incNumElements(void) + incNumElements() { ++_numElements; } @@ -175,13 +175,13 @@ public: startWord(const vespalib::stringref &word); void - endWord(void); + endWord(); void startDocument(uint32_t docId); void - endDocument(void); + endDocument(); void startElement(uint32_t elementId, @@ -189,31 +189,31 @@ public: uint32_t elementLen); void - endElement(void); + endElement(); void addOcc(const WordDocElementWordPosFeatures &features); void - setValid(void) + setValid() { _valid = true; } bool - getValid(void) const + getValid() const { return _valid; } const Schema::IndexField & - getSchemaField(void); + getSchemaField(); const vespalib::string & - getName(void); + getName(); vespalib::string - getDir(void); + getDir(); void open(uint32_t docIdLimit, uint64_t numWordIds, @@ -221,10 +221,10 @@ public: const FileHeaderContext &fileHeaderContext); void - close(void); + close(); uint32_t - getIndexId(void) const + getIndexId() const { return _fieldId; } @@ -250,7 +250,7 @@ public: appendFeatures(DocIdAndFeatures &features); bool - isValid(void) const + isValid() const { return _dFeatures != _dFeaturesE; } @@ -268,14 +268,14 @@ public: } -FileHandle::FileHandle(void) +FileHandle::FileHandle() : _fieldWriter(NULL), _docIdAndFeatures() { } -FileHandle::~FileHandle(void) +FileHandle::~FileHandle() { delete _fieldWriter; } @@ -307,7 +307,7 @@ FileHandle::open(const vespalib::stringref &dir, void -FileHandle::close(void) +FileHandle::close() { bool ret = true; if (_fieldWriter != NULL) { @@ -344,7 +344,7 @@ IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, } -IndexBuilder::FieldHandle::~FieldHandle(void) +IndexBuilder::FieldHandle::~FieldHandle() { } @@ -389,7 +389,7 @@ IndexBuilder::FieldHandle::startDocument(uint32_t docId) void -IndexBuilder::FieldHandle::endDocument(void) +IndexBuilder::FieldHandle::endDocument() { assert(_docRef != noDocRef()); assert(_elRef == noElRef()); @@ -423,7 +423,7 @@ startElement(uint32_t elementId, void -IndexBuilder::FieldHandle::endElement(void) +IndexBuilder::FieldHandle::endElement() { assert(_elRef != noElRef()); FHWordDocElementFeatures &ef = _wdfef[_elRef]; @@ -451,14 +451,14 @@ addOcc(const WordDocElementWordPosFeatures &features) const Schema::IndexField & -IndexBuilder::FieldHandle::getSchemaField(void) +IndexBuilder::FieldHandle::getSchemaField() { return _schema->getIndexField(_fieldId); } const vespalib::string & -IndexBuilder::FieldHandle::getName(void) +IndexBuilder::FieldHandle::getName() { return getSchemaField().getName(); @@ -466,7 +466,7 @@ IndexBuilder::FieldHandle::getName(void) vespalib::string -IndexBuilder::FieldHandle::getDir(void) +IndexBuilder::FieldHandle::getDir() { return _ib->appendToPrefix(getName()); } @@ -484,7 +484,7 @@ IndexBuilder::FieldHandle::open(uint32_t docIdLimit, uint64_t numWordIds, void -IndexBuilder::FieldHandle::close(void) +IndexBuilder::FieldHandle::close() { _files.close(); } @@ -551,7 +551,7 @@ IndexBuilder::IndexBuilder(const Schema &schema) } -IndexBuilder::~IndexBuilder(void) +IndexBuilder::~IndexBuilder() { } @@ -569,7 +569,7 @@ IndexBuilder::startWord(const vespalib::stringref &word) void -IndexBuilder::endWord(void) +IndexBuilder::endWord() { assert(_inWord); assert(_currentField != NULL); @@ -593,7 +593,7 @@ IndexBuilder::startDocument(uint32_t docId) void -IndexBuilder::endDocument(void) +IndexBuilder::endDocument() { assert(_curDocId != noDocId()); assert(_currentField != NULL); @@ -616,7 +616,7 @@ IndexBuilder::startField(uint32_t fieldId) void -IndexBuilder::endField(void) +IndexBuilder::endField() { assert(_curDocId == noDocId()); assert(!_inWord); @@ -637,7 +637,7 @@ IndexBuilder::startElement(uint32_t elementId, void -IndexBuilder::endElement(void) +IndexBuilder::endElement() { assert(_currentField != NULL); _currentField->endElement(); @@ -698,7 +698,7 @@ IndexBuilder::open(uint32_t docIdLimit, uint64_t numWordIds, void -IndexBuilder::close(void) +IndexBuilder::close() { // TODO: Filter for text indexes for (FieldHandle & fh : _fields) { diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h index 88c64ddf0df..44d6b11401d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h @@ -36,11 +36,11 @@ private: const Schema &_schema; // Ptr to allow being std::vector member - static uint32_t noDocId(void) { + static uint32_t noDocId() { return std::numeric_limits<uint32_t>::max(); } - static uint64_t noWordNumHigh(void) { + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } @@ -49,16 +49,16 @@ public: // schema argument must live until indexbuilder has been deleted. IndexBuilder(const Schema &schema); - virtual ~IndexBuilder(void); - - virtual void startWord(const vespalib::stringref &word) override; - virtual void endWord(void) override; - virtual void startDocument(uint32_t docId) override; - virtual void endDocument(void) override; - virtual void startField(uint32_t fieldId) override; - virtual void endField(void) override; + virtual ~IndexBuilder(); + + virtual void startWord(const vespalib::stringref &word) override; + virtual void endWord() override; + virtual void startDocument(uint32_t docId) override; + virtual void endDocument() override; + virtual void startField(uint32_t fieldId) override; + virtual void endField() override; virtual void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) override; - virtual void endElement(void) override; + virtual void endElement() override; virtual void addOcc(const WordDocElementWordPosFeatures &features) override; // TODO: methods for attribute vectors. @@ -74,7 +74,7 @@ public: const TuneFileIndexing &tuneFileIndexing, const search::common::FileHeaderContext &fileHandleContext); - void close(void); + void close(); }; } // namespace diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 348262f1d0e..c3bb41e49e3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -1,17 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.pagedict4file"); #include "pagedict4file.h" +#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/io/fileutil.h> -namespace -{ +#include <vespa/log/log.h> +LOG_SETUP(".diskindex.pagedict4file"); + +namespace { vespalib::string myPId("PageDict4P.1"); vespalib::string mySPId("PageDict4SP.1"); @@ -35,20 +34,15 @@ using search::common::FileHeaderContext; using search::index::PostingListParams; using vespalib::getLastErrorString; -namespace search -{ +namespace search::diskindex { -namespace diskindex -{ - -namespace -{ +namespace { const uint32_t headerAlign = 4096; } -PageDict4FileSeqRead::PageDict4FileSeqRead(void) +PageDict4FileSeqRead::PageDict4FileSeqRead() : _pReader(NULL), _ssReader(NULL), _ssd(), @@ -78,7 +72,7 @@ PageDict4FileSeqRead::PageDict4FileSeqRead(void) } -PageDict4FileSeqRead::~PageDict4FileSeqRead(void) +PageDict4FileSeqRead::~PageDict4FileSeqRead() { delete _pReader; delete _ssReader; @@ -120,7 +114,7 @@ PageDict4FileSeqRead::readSSHeader() void -PageDict4FileSeqRead::readSPHeader(void) +PageDict4FileSeqRead::readSPHeader() { DC &spd = _spd; @@ -145,7 +139,7 @@ PageDict4FileSeqRead::readSPHeader(void) void -PageDict4FileSeqRead::readPHeader(void) +PageDict4FileSeqRead::readPHeader() { DC &pd = _pd; @@ -284,7 +278,7 @@ PageDict4FileSeqRead::open(const vespalib::string &name, bool -PageDict4FileSeqRead::close(void) +PageDict4FileSeqRead::close() { delete _pReader; delete _ssReader; @@ -350,7 +344,7 @@ PageDict4FileSeqRead::getParams(PostingListParams ¶ms) } -PageDict4FileSeqWrite::PageDict4FileSeqWrite(void) +PageDict4FileSeqWrite::PageDict4FileSeqWrite() : _pWriter(NULL), _spWriter(NULL), _ssWriter(NULL), @@ -373,7 +367,7 @@ PageDict4FileSeqWrite::PageDict4FileSeqWrite(void) } -PageDict4FileSeqWrite::~PageDict4FileSeqWrite(void) +PageDict4FileSeqWrite::~PageDict4FileSeqWrite() { delete _pWriter; delete _spWriter; @@ -484,7 +478,7 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, bool -PageDict4FileSeqWrite::close(void) +PageDict4FileSeqWrite::close() { _pWriter->flush(); uint64_t usedPBits = _pe.getWriteOffset(); @@ -747,6 +741,4 @@ PageDict4FileSeqWrite::getParams(PostingListParams ¶ms) params.set("numWordIds", _sse._numWordIds); } -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index bc080fc58c3..574269c51c5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -5,19 +5,11 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/pagedict4.h> +#include <vespa/fastos/file.h> -namespace vespalib -{ - -class GenericHeader; - -} +namespace vespalib { class GenericHeader; } -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { /** * Dictionary file containing words and counts for words. @@ -64,17 +56,17 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead readSSHeader(); void - readSPHeader(void); + readSPHeader(); void - readPHeader(void); + readPHeader(); public: - PageDict4FileSeqRead(void); + PageDict4FileSeqRead(); virtual - ~PageDict4FileSeqRead(void); + ~PageDict4FileSeqRead(); /** * Read word and counts. Only nonzero counts are returned. If at @@ -91,7 +83,7 @@ public: /** * Close dictionary file. */ - virtual bool close(void) override; + virtual bool close() override; /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -177,10 +169,10 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite updateSSHeader(uint64_t fileBitSize); public: - PageDict4FileSeqWrite(void); + PageDict4FileSeqWrite(); virtual - ~PageDict4FileSeqWrite(void); + ~PageDict4FileSeqWrite(); /** * Write word and counts. Only nonzero counts should be supplied. @@ -202,7 +194,7 @@ public: * Close dictionary file. */ virtual bool - close(void) override; + close() override; /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -232,8 +224,4 @@ public: getParams(index::PostingListParams ¶ms) override; }; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp index e43e10f0709..eb064307cc2 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp @@ -23,7 +23,7 @@ namespace search { namespace diskindex { -PageDict4RandRead::PageDict4RandRead(void) +PageDict4RandRead::PageDict4RandRead() : DictionaryFileRandRead(), _ssReader(), _ssd(), @@ -80,7 +80,7 @@ PageDict4RandRead::readSSHeader() void -PageDict4RandRead::readSPHeader(void) +PageDict4RandRead::readSPHeader() { DC d; ComprFileReadContext rc(d); @@ -113,7 +113,7 @@ PageDict4RandRead::readSPHeader(void) void -PageDict4RandRead::readPHeader(void) +PageDict4RandRead::readPHeader() { DC d; ComprFileReadContext rc(d); @@ -254,7 +254,7 @@ PageDict4RandRead::open(const vespalib::string &name, bool -PageDict4RandRead::close(void) +PageDict4RandRead::close() { _ssReader.reset(); @@ -268,7 +268,7 @@ PageDict4RandRead::close(void) uint64_t -PageDict4RandRead::getNumWordIds(void) const +PageDict4RandRead::getNumWordIds() const { return _ssd._numWordIds; } diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp index 3a66fc05685..bd46caf88b5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp @@ -1,19 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.wordnummapper"); +#include "wordnummapper.h" #include <vespa/vespalib/objects/nbostream.h> #include <vespa/fastlib/io/bufferedfile.h> -#include "wordnummapper.h" - -namespace search -{ -namespace diskindex -{ +namespace search::diskindex { -WordNumMapping::WordNumMapping(void) +WordNumMapping::WordNumMapping() : _old2newwords(), _oldDictSize(0u) { @@ -46,7 +39,7 @@ WordNumMapping::readMappingFile(const vespalib::string &name, void -WordNumMapping::noMappingFile(void) +WordNumMapping::noMappingFile() { Array &map = _old2newwords; map.resize(2); @@ -57,7 +50,7 @@ WordNumMapping::noMappingFile(void) void -WordNumMapping::clear(void) +WordNumMapping::clear() { Array &map = _old2newwords; map.clear(); @@ -90,7 +83,7 @@ WordNumMapper::sanityCheck(bool allowHoles) uint64_t -WordNumMapping::getMaxMappedWordNum(void) const +WordNumMapping::getMaxMappedWordNum() const { WordNumMapper mapper(*this); return mapper.getMaxMappedWordNum(); @@ -104,7 +97,4 @@ WordNumMapping::sanityCheck(bool allowHoles) mapper.sanityCheck(allowHoles); } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h index d12e0211b94..a45f5dc64ed 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/array.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/string.h> -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { class WordNumMapper; @@ -16,82 +13,50 @@ class WordNumMapping { typedef vespalib::Array<uint64_t> Array; - static uint64_t - noWordNumHigh(void) - { + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } - static uint64_t - noWordNum(void) - { - return 0u; - } + static uint64_t noWordNum() { return 0u; } Array _old2newwords; uint64_t _oldDictSize; public: - WordNumMapping(void); + WordNumMapping(); - const uint64_t * - getOld2NewWordNums(void) const - { + const uint64_t *getOld2NewWordNums() const { return (_old2newwords.empty()) ? NULL : &_old2newwords[0]; } - uint64_t - getOldDictSize(void) const - { - return _oldDictSize; - } - - void - readMappingFile(const vespalib::string &name, - const TuneFileSeqRead &tuneFileRead); - - void - noMappingFile(void); - - void - clear(void); - - void - setup(uint32_t numWordIds); - - uint64_t - getMaxMappedWordNum(void) const; - - void - sanityCheck(bool allowHoles); + uint64_t getOldDictSize() const { return _oldDictSize; } + void readMappingFile(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead); + void noMappingFile(); + void clear(); + void setup(uint32_t numWordIds); + uint64_t getMaxMappedWordNum() const; + void sanityCheck(bool allowHoles); }; class WordNumMapper { - static uint64_t - noWordNumHigh(void) - { + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } - static uint64_t - noWordNum(void) - { - return 0u; - } + static uint64_t noWordNum() { return 0u; } const uint64_t *_old2newwords; uint64_t _oldDictSize; public: - WordNumMapper(void) + WordNumMapper() : _old2newwords(NULL), _oldDictSize(0) - { - } + {} WordNumMapper(const WordNumMapping &mapping) : _old2newwords(NULL), @@ -100,38 +65,20 @@ public: setup(mapping); } - void - setup(const WordNumMapping &mapping) - { + void setup(const WordNumMapping &mapping) { _old2newwords = mapping.getOld2NewWordNums(); _oldDictSize = mapping.getOldDictSize(); } - uint64_t - map(uint32_t wordNum) const - { + uint64_t map(uint32_t wordNum) const { return (_old2newwords != NULL) ? _old2newwords[wordNum] : wordNum; } - uint64_t - getMaxWordNum(void) const - { - return _oldDictSize; - } - - uint64_t - getMaxMappedWordNum(void) const - { - return map(_oldDictSize); - } - - void - sanityCheck(bool allowHoles); + uint64_t getMaxWordNum() const { return _oldDictSize; } + uint64_t getMaxMappedWordNum() const { return map(_oldDictSize); } + void sanityCheck(bool allowHoles); }; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp new file mode 100644 index 00000000000..937bef411db --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zcbuf.h" +#include <cstdlib> +#include <cstring> + +namespace search::diskindex { + +ZcBuf::ZcBuf() + : _valI(NULL), + _valE(NULL), + _mallocStart(NULL), + _mallocSize(0) +{ +} + +ZcBuf::~ZcBuf() +{ + free(_mallocStart); +} + +void +ZcBuf::clearReserve(size_t reserveSize) +{ + if (reserveSize + zcSlack() > _mallocSize) { + size_t newSize = _mallocSize * 2; + if (newSize < 16) + newSize = 16; + while (newSize < reserveSize + zcSlack()) + newSize *= 2; + uint8_t *newBuf = static_cast<uint8_t *>(malloc(newSize)); + free(_mallocStart); + _mallocStart = newBuf; + _mallocSize = newSize; + } + _valE = _mallocStart + _mallocSize - zcSlack(); + _valI = _mallocStart; +} + + +void +ZcBuf::expand() +{ + size_t newSize = _mallocSize * 2; + size_t oldSize = size(); + if (newSize < 16) + newSize = 16; + + uint8_t *newBuf = static_cast<uint8_t *>(malloc(newSize)); + + if (oldSize > 0) + memcpy(newBuf, _mallocStart, oldSize); + free(_mallocStart); + _mallocStart = newBuf; + _mallocSize = newSize; + _valI = _mallocStart + oldSize; + _valE = _mallocStart + newSize - zcSlack(); +} + +} // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcbuf.h b/searchlib/src/vespa/searchlib/diskindex/zcbuf.h new file mode 100644 index 00000000000..71bc5970cc7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zcbuf.h @@ -0,0 +1,84 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <cstddef> + +namespace search::diskindex { + +/* + * Class containing Zc-encoded data in a memory buffer, typically + * docid deltas and skip information for posting lists. + */ +class ZcBuf +{ +public: + uint8_t *_valI; + uint8_t *_valE; + uint8_t *_mallocStart; + size_t _mallocSize; + + ZcBuf(); + ~ZcBuf(); + + static size_t zcSlack() { return 4; } + void clearReserve(size_t reserveSize); + void clear() { _valI = _mallocStart; } + size_t capacity() const { return _valE - _mallocStart; } + size_t size() const { return _valI - _mallocStart; } + size_t pos() const { return _valI - _mallocStart; } + void expand(); + + void maybeExpand() { + if (__builtin_expect(_valI >= _valE, false)) + expand(); + } + + void encode(uint32_t num) { + for (;;) { + if (num < (1 << 7)) { + *_valI++ = num; + break; + } + *_valI++ = (num & ((1 << 7) - 1)) | (1 << 7); + num >>= 7; + } + maybeExpand(); + } + + uint32_t decode() { + uint32_t res; + uint8_t *valI = _valI; + if (__builtin_expect(valI[0] < (1 << 7), true)) { + res = valI[0]; + valI += 1; + } else if (__builtin_expect(valI[1] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + (valI[1] << 7); + valI += 2; + } else if (__builtin_expect(valI[2] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + (valI[2] << 14); + valI += 3; + } else if (__builtin_expect(valI[3] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + ((valI[2] & ((1 << 7) - 1)) << 14) + + (valI[3] << 21); + valI += 4; + } else { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + ((valI[2] & ((1 << 7) - 1)) << 14) + + ((valI[3] & ((1 << 7) - 1)) << 21) + + (valI[4] << 28); + valI += 5; + } + _valI = valI; + return res; + } +}; + +} // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp index 440a61fcab2..18e34b8ef74 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp @@ -1,20 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "zcposocc.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/docidandfeatures.h> -LOG_SETUP(".diskindex.zcposocc"); -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { using search::bitcompression::PosOccFieldsParams; using search::bitcompression::EG2PosOccDecodeContext; @@ -55,7 +48,7 @@ setFeatureParams(const PostingListParams ¶ms) const vespalib::string & -Zc4PosOccSeqRead::getSubIdentifier(void) +Zc4PosOccSeqRead::getSubIdentifier() { PosOccFieldsParams fieldsParams; EG2PosOccDecodeContext<true> d(&fieldsParams); @@ -110,7 +103,7 @@ setFeatureParams(const PostingListParams ¶ms) const vespalib::string & -ZcPosOccSeqRead::getSubIdentifier(void) +ZcPosOccSeqRead::getSubIdentifier() { PosOccFieldsParams fieldsParams; EGPosOccDecodeContext<true> d(&fieldsParams); @@ -131,7 +124,4 @@ ZcPosOccSeqWrite::ZcPosOccSeqWrite(const Schema &schema, _fieldsParams.setSchemaParams(schema, indexId); } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.h b/searchlib/src/vespa/searchlib/diskindex/zcposocc.h index 3d5936210f8..b12c3b9eed9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.h @@ -45,7 +45,7 @@ private: public: ZcPosOccSeqRead(index::PostingListCountFileSeqRead *countFile); void setFeatureParams(const PostingListParams ¶ms) override; - static const vespalib::string &getSubIdentifier(void); + static const vespalib::string &getSubIdentifier(); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index fdb498e8e28..b1f73dc2088 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -1,15 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.zcposocciterators"); #include "zcposocciterators.h" -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { using search::fef::TermFieldMatchDataArray; using search::bitcompression::PosOccFieldsParams; @@ -26,7 +19,7 @@ Zc4RareWordPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimi : Zc4RareWordPostingIterator<bigEndian>(matchData, start, docIdLimit), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - LOG_ASSERT(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); _decodeContext = &_decodeContextReal; } @@ -40,7 +33,7 @@ Zc4PosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, : ZcPostingIterator<bigEndian>(minChunkDocs, false, counts, matchData, start, docIdLimit), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - LOG_ASSERT(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); _decodeContext = &_decodeContextReal; } @@ -53,7 +46,7 @@ ZcRareWordPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit : ZcRareWordPostingIterator<bigEndian>(matchData, start, docIdLimit), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - LOG_ASSERT(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); _decodeContext = &_decodeContextReal; } @@ -67,7 +60,7 @@ ZcPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, : ZcPostingIterator<bigEndian>(minChunkDocs, true, counts, matchData, start, docIdLimit), _decodeContextReal(start.getOccurences(), start.getBitOffset(), bitLength, fieldsParams) { - LOG_ASSERT(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); + assert(!matchData.valid() || (fieldsParams->getNumFields() == matchData.size())); _decodeContext = &_decodeContextReal; } @@ -84,6 +77,4 @@ template class ZcRareWordPosOccIterator<false>; template class ZcPosOccIterator<true>; template class ZcPosOccIterator<false>; -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h index bab8b05fb4d..d19ca4ec9ed 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h @@ -64,7 +64,7 @@ public: class Zc4PosOccRandRead : public ZcPosOccRandRead { public: - Zc4PosOccRandRead(void); + Zc4PosOccRandRead(); /** * Create iterator for single word. Semantic lifetime of counts and diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index e416dcf3e61..a1b305f43dc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -1,20 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "zcposting.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistcountfile.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/fileheader.h> -#include <vespa/searchlib/common/fileheadercontext.h> +#include <vespa/log/log.h> LOG_SETUP(".diskindex.zcposting"); -namespace -{ +namespace { vespalib::string myId5("Zc.5"); vespalib::string myId4("Zc.4"); @@ -22,11 +20,7 @@ vespalib::string emptyId; } -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { using index::PostingListCountFileSeqRead; using index::PostingListCountFileSeqWrite; @@ -37,45 +31,6 @@ using vespalib::nbostream; using vespalib::getLastErrorString; -void -ZcBuf::clearReserve(size_t reserveSize) -{ - if (reserveSize + zcSlack() > _mallocSize) { - size_t newSize = _mallocSize * 2; - if (newSize < 16) - newSize = 16; - while (newSize < reserveSize + zcSlack()) - newSize *= 2; - uint8_t *newBuf = static_cast<uint8_t *>(malloc(newSize)); - free(_mallocStart); - _mallocStart = newBuf; - _mallocSize = newSize; - } - _valE = _mallocStart + _mallocSize - zcSlack(); - _valI = _mallocStart; -} - - -void -ZcBuf::expand(void) -{ - size_t newSize = _mallocSize * 2; - size_t oldSize = size(); - if (newSize < 16) - newSize = 16; - - uint8_t *newBuf = static_cast<uint8_t *>(malloc(newSize)); - - if (oldSize > 0) - memcpy(newBuf, _mallocStart, oldSize); - free(_mallocStart); - _mallocStart = newBuf; - _mallocSize = newSize; - _valI = _mallocStart + oldSize; - _valE = _mallocStart + newSize - zcSlack(); -} - - Zc4PostingSeqRead:: Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) : PostingListFileSeqRead(), @@ -138,7 +93,7 @@ Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) } -Zc4PostingSeqRead::~Zc4PostingSeqRead(void) +Zc4PostingSeqRead::~Zc4PostingSeqRead() { } @@ -302,7 +257,7 @@ Zc4PostingSeqRead::checkPointRead(nbostream &in) void -Zc4PostingSeqRead::readWordStartWithSkip(void) +Zc4PostingSeqRead::readWordStartWithSkip() { typedef FeatureEncodeContextBE EC; DecodeContext &d = *_decodeContext; @@ -486,7 +441,7 @@ Zc4PostingSeqRead::readWordStartWithSkip(void) void -Zc4PostingSeqRead::readWordStart(void) +Zc4PostingSeqRead::readWordStart() { typedef FeatureEncodeContextBE EC; UC64_DECODECONTEXT_CONSTRUCTOR(o, _decodeContext->_); @@ -591,7 +546,7 @@ Zc4PostingSeqRead::open(const vespalib::string &name, bool -Zc4PostingSeqRead::close(void) +Zc4PostingSeqRead::close() { _readContext.dropComprBuf(); _file.Close(); @@ -630,7 +585,7 @@ Zc4PostingSeqRead::getFeatureParams(PostingListParams ¶ms) void -Zc4PostingSeqRead::readHeader(void) +Zc4PostingSeqRead::readHeader() { FeatureDecodeContextBE &d = *_decodeContext; const vespalib::string &myId = _dynamicK ? myId5 : myId4; @@ -672,14 +627,14 @@ Zc4PostingSeqRead::readHeader(void) const vespalib::string & -Zc4PostingSeqRead::getIdentifier(void) +Zc4PostingSeqRead::getIdentifier() { return myId4; } uint64_t -Zc4PostingSeqRead::getCurrentPostingOffset(void) const +Zc4PostingSeqRead::getCurrentPostingOffset() const { FeatureDecodeContextBE &d = *_decodeContext; return d.getReadOffset() - _headerBitLen; @@ -743,7 +698,7 @@ Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) } -Zc4PostingSeqWrite::~Zc4PostingSeqWrite(void) +Zc4PostingSeqWrite::~Zc4PostingSeqWrite() { } @@ -765,7 +720,7 @@ writeDocIdAndFeatures(const DocIdAndFeatures &features) void -Zc4PostingSeqWrite::flushWord(void) +Zc4PostingSeqWrite::flushWord() { if (__builtin_expect(_docIds.size() >= _minSkipDocs || !_counts._segments.empty(), false)) { @@ -904,7 +859,7 @@ Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) void -Zc4PostingSeqWrite::updateHeader(void) +Zc4PostingSeqWrite::updateHeader() { vespalib::FileHeader h; FastOS_File f; @@ -978,7 +933,7 @@ Zc4PostingSeqWrite::open(const vespalib::string &name, bool -Zc4PostingSeqWrite::close(void) +Zc4PostingSeqWrite::close() { EncodeContext &e = _encodeContext; @@ -1059,7 +1014,7 @@ getFeatureParams(PostingListParams ¶ms) void -Zc4PostingSeqWrite::flushChunk(void) +Zc4PostingSeqWrite::flushChunk() { /* TODO: Flush chunk and prepare for new (possible short) chunk */ flushWordWithSkip(true); @@ -1072,7 +1027,7 @@ Zc4PostingSeqWrite::flushChunk(void) void -Zc4PostingSeqWrite::calcSkipInfo(void) +Zc4PostingSeqWrite::calcSkipInfo() { uint32_t lastDocId = 0u; uint32_t lastL1SkipDocId = 0u; @@ -1322,7 +1277,7 @@ Zc4PostingSeqWrite::flushWordWithSkip(bool hasMore) void -Zc4PostingSeqWrite::flushWordNoSkip(void) +Zc4PostingSeqWrite::flushWordNoSkip() { // Too few document ids for skip info. assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); @@ -1357,7 +1312,7 @@ Zc4PostingSeqWrite::flushWordNoSkip(void) void -Zc4PostingSeqWrite::resetWord(void) +Zc4PostingSeqWrite::resetWord() { _docIds.clear(); _encodeFeatures->setupWrite(_featureWriteContext); @@ -1417,7 +1372,7 @@ readDocIdAndFeatures(DocIdAndFeatures &features) const vespalib::string & -ZcPostingSeqRead::getIdentifier(void) +ZcPostingSeqRead::getIdentifier() { return myId5; } @@ -1431,7 +1386,7 @@ ZcPostingSeqWrite::ZcPostingSeqWrite(PostingListCountFileSeqWrite *countFile) void -ZcPostingSeqWrite::flushWordNoSkip(void) +ZcPostingSeqWrite::flushWordNoSkip() { // Too few document ids for skip info. assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); @@ -1466,14 +1421,11 @@ ZcPostingSeqWrite::flushWordNoSkip(void) resetWord(); } - -} // namespace diskindex - -} // namespace search +} // namespace search::diskindex #include <vespa/vespalib/objects/nbostream.hpp> namespace vespalib { using UInt32Pair = std::pair<uint32_t, uint32_t>; template nbostream &nbostream::saveVector<UInt32Pair>(const std::vector<UInt32Pair> &); template nbostream &nbostream::restoreVector<UInt32Pair>(std::vector<UInt32Pair> &); -}
\ No newline at end of file +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 8ec0f2e2697..4f6591388e0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -2,8 +2,10 @@ #pragma once +#include "zcbuf.h" #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/bitcompression/compression.h> +#include <vespa/fastos/file.h> namespace search { @@ -16,82 +18,6 @@ class PostingListCountFileSeqWrite; namespace diskindex { -class ZcBuf -{ -public: - uint8_t *_valI; - uint8_t *_valE; - uint8_t *_mallocStart; - size_t _mallocSize; - - ZcBuf() - : _valI(NULL), - _valE(NULL), - _mallocStart(NULL), - _mallocSize(0) - {} - - ~ZcBuf() { free(_mallocStart); } - - static size_t zcSlack() { return 4; } - void clearReserve(size_t reserveSize); - void clear() { _valI = _mallocStart; } - size_t capacity() const { return _valE - _mallocStart; } - size_t size() const { return _valI - _mallocStart; } - size_t pos() const { return _valI - _mallocStart; } - void expand(); - - void maybeExpand() { - if (__builtin_expect(_valI >= _valE, false)) - expand(); - } - - void encode(uint32_t num) { - for (;;) { - if (num < (1 << 7)) { - *_valI++ = num; - break; - } - *_valI++ = (num & ((1 << 7) - 1)) | (1 << 7); - num >>= 7; - } - maybeExpand(); - } - - uint32_t decode() { - uint32_t res; - uint8_t *valI = _valI; - if (__builtin_expect(valI[0] < (1 << 7), true)) { - res = valI[0]; - valI += 1; - } else if (__builtin_expect(valI[1] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - (valI[1] << 7); - valI += 2; - } else if (__builtin_expect(valI[2] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - (valI[2] << 14); - valI += 3; - } else if (__builtin_expect(valI[3] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - ((valI[2] & ((1 << 7) - 1)) << 14) + - (valI[3] << 21); - valI += 4; - } else { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - ((valI[2] & ((1 << 7) - 1)) << 14) + - ((valI[3] & ((1 << 7) - 1)) << 21) + - (valI[4] << 28); - valI += 5; - } - _valI = valI; - return res; - } -}; - class Zc4PostingSeqRead : public index::PostingListFileSeqRead { Zc4PostingSeqRead(const Zc4PostingSeqRead &); diff --git a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp b/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp index 1a2b8f090e1..fb65f1343b4 100644 --- a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp +++ b/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "bytecomplens.h" + #include <vespa/log/log.h> LOG_SETUP(".search.docstore"); -#include "bytecomplens.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/cachestats.h b/searchlib/src/vespa/searchlib/docstore/cachestats.h index 5f4e690ce8c..bf11397cfa8 100644 --- a/searchlib/src/vespa/searchlib/docstore/cachestats.h +++ b/searchlib/src/vespa/searchlib/docstore/cachestats.h @@ -13,7 +13,7 @@ struct CacheStats { size_t elements; size_t memory_used; - CacheStats(void) + CacheStats() : hits(0), misses(0), elements(0), diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 1b243e1546e..cddf8c96b2e 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/docstore/chunkformats.h> +#include "chunkformats.h" #include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/stringfmt.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp index bdff46aacbc..0843d473121 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/docstore/chunkformats.h> +#include "chunkformats.h" #include <vespa/vespalib/util/crc.h> #include <vespa/vespalib/xxhash/xxhash.h> +#include <vespa/vespalib/util/stringfmt.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_id.cpp b/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_id.cpp index 50177e5af31..3a1c0d55d18 100644 --- a/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_id.cpp +++ b/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_id.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "data_store_file_chunk_id.h" #include "filechunk.h" -namespace search -{ +namespace search { vespalib::string DataStoreFileChunkId::createName(const vespalib::string &baseName) const diff --git a/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_stats.h b/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_stats.h index 48a99c0762c..b7eb6797f7f 100644 --- a/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_stats.h +++ b/searchlib/src/vespa/searchlib/docstore/data_store_file_chunk_stats.h @@ -18,10 +18,11 @@ public: double maxBucketSpread_in, uint64_t lastSerialNum_in, uint64_t lastFlushedSerialNum_in, + uint32_t docIdLimit_in, uint64_t nameId_in) : DataStoreStorageStats(diskUsage_in, diskBloat_in, - maxBucketSpread_in, - lastSerialNum_in, lastFlushedSerialNum_in), + maxBucketSpread_in, lastSerialNum_in, + lastFlushedSerialNum_in, docIdLimit_in), DataStoreFileChunkId(nameId_in) { } diff --git a/searchlib/src/vespa/searchlib/docstore/data_store_storage_stats.h b/searchlib/src/vespa/searchlib/docstore/data_store_storage_stats.h index db984564775..19390d51ed8 100644 --- a/searchlib/src/vespa/searchlib/docstore/data_store_storage_stats.h +++ b/searchlib/src/vespa/searchlib/docstore/data_store_storage_stats.h @@ -16,20 +16,23 @@ class DataStoreStorageStats double _maxBucketSpread; uint64_t _lastSerialNum; uint64_t _lastFlushedSerialNum; + uint32_t _docIdLimit; public: DataStoreStorageStats(uint64_t diskUsage_in, uint64_t diskBloat_in, double maxBucketSpread_in, - uint64_t lastSerialNum_in, uint64_t lastFlushedSerialNum_in) + uint64_t lastSerialNum_in, uint64_t lastFlushedSerialNum_in, uint32_t docIdLimit_in) : _diskUsage(diskUsage_in), _diskBloat(diskBloat_in), _maxBucketSpread(maxBucketSpread_in), _lastSerialNum(lastSerialNum_in), - _lastFlushedSerialNum(lastFlushedSerialNum_in) + _lastFlushedSerialNum(lastFlushedSerialNum_in), + _docIdLimit(docIdLimit_in) { } uint64_t diskUsage() const { return _diskUsage; } uint64_t diskBloat() const { return _diskBloat; } double maxBucketSpread() const { return _maxBucketSpread; } uint64_t lastSerialNum() const { return _lastSerialNum; } uint64_t lastFlushedSerialNum() const { return _lastFlushedSerialNum; } + uint32_t docIdLimit() const { return _docIdLimit; } }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/document_store_visitor_progress.cpp b/searchlib/src/vespa/searchlib/docstore/document_store_visitor_progress.cpp index cca794ccb69..adde03d8ff0 100644 --- a/searchlib/src/vespa/searchlib/docstore/document_store_visitor_progress.cpp +++ b/searchlib/src/vespa/searchlib/docstore/document_store_visitor_progress.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_store_visitor_progress.h" -namespace search -{ - +namespace search { DocumentStoreVisitorProgress::DocumentStoreVisitorProgress() : search::IDocumentStoreVisitorProgress(), @@ -13,19 +10,16 @@ DocumentStoreVisitorProgress::DocumentStoreVisitorProgress() { } - void DocumentStoreVisitorProgress::updateProgress(double progress) { _progress = progress; } - double DocumentStoreVisitorProgress::getProgress() const { return _progress; } - -} // namespace proton +} diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index ceb82a60cbf..fd7df29cd49 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -4,9 +4,9 @@ #include "documentstore.h" #include "visitcache.h" #include "ibucketizer.h" -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/stllike/cache.hpp> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/document/util/compressor.h> namespace search { @@ -517,6 +517,12 @@ DocumentStore::canShrinkLidSpace() const return _backingStore.canShrinkLidSpace(); } +size_t +DocumentStore::getEstimatedShrinkLidSpaceGain() const +{ + return _backingStore.getEstimatedShrinkLidSpaceGain(); +} + void DocumentStore::shrinkLidSpace() { diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 93bf899fbb6..14362ddaf5e 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -105,9 +105,8 @@ public: /** * Get the number of entries (including removed IDs * or gaps in the local ID sequence) in the document store. - * @return The next local ID expected to be used. */ - uint64_t nextId() const override { return _backingStore.nextId(); } + uint32_t getDocIdLimit() const override { return _backingStore.getDocIdLimit(); } /** * Calculate memory used by this instance. During flush() actual @@ -166,9 +165,10 @@ public: /** * Implements common::ICompactableLidSpace */ - void compactLidSpace(uint32_t wantedDocLidLimit) override; - bool canShrinkLidSpace() const override; - void shrinkLidSpace() override; + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; + virtual bool canShrinkLidSpace() const override; + virtual size_t getEstimatedShrinkLidSpaceGain() const override; + virtual void shrinkLidSpace() override; private: bool useCache() const; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index f9f102ef752..746ec875781 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -23,6 +23,7 @@ namespace { constexpr size_t ALIGNMENT=0x1000; constexpr size_t ENTRY_BIAS_SIZE=8; +const vespalib::string DOC_ID_LIMIT_KEY("docIdLimit"); } @@ -75,6 +76,7 @@ FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & base _dataHeaderLen(0u), _idxHeaderLen(0u), _lastPersistedSerialNum(0), + _docIdLimit(std::numeric_limits<uint32_t>::max()), _modificationTime() { FastOS_File dataFile(_dataFileName.c_str()); @@ -153,7 +155,7 @@ FileChunk::erase() } size_t -FileChunk::updateLidMap(const LockGuard & guard, ISetLid & ds, uint64_t serialNum) +FileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit) { size_t sz(0); assert(_chunkInfo.empty()); @@ -164,7 +166,7 @@ FileChunk::updateLidMap(const LockGuard & guard, ISetLid & ds, uint64_t serialNu if (idxFile.IsMemoryMapped()) { const int64_t fileSize = idxFile.GetSize(); if (_idxHeaderLen == 0) { - _idxHeaderLen = readIdxHeader(idxFile); + _idxHeaderLen = readIdxHeader(idxFile, _docIdLimit); } vespalib::nbostream is(static_cast<const char *>(idxFile.MemoryMapPtr(0)) + _idxHeaderLen, fileSize - _idxHeaderLen); @@ -213,12 +215,16 @@ FileChunk::updateLidMap(const LockGuard & guard, ISetLid & ds, uint64_t serialNu BucketDensityComputer bucketMap(_bucketizer); for (size_t i(0), m(chunkMeta.getNumEntries()); i < m; i++) { const LidMeta & lidMeta(chunkMeta[i]); - if (_bucketizer && (lidMeta.size() > 0)) { - document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid()); - bucketMap.recordLid(bucketId); - globalBucketMap.recordLid(bucketId); + if (lidMeta.getLid() < docIdLimit) { + if (_bucketizer && (lidMeta.size() > 0)) { + document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid()); + bucketMap.recordLid(bucketId); + globalBucketMap.recordLid(bucketId); + } + ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size())); + } else { + remove(lidMeta.getLid(), lidMeta.size()); } - ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size())); _addedBytes += adjustSize(lidMeta.size()); } serialNum = chunkMeta.getLastSerial(); @@ -392,7 +398,7 @@ FileChunk::readDataHeader(FileRandRead &datFile) uint64_t -FileChunk::readIdxHeader(FastOS_FileInterface &idxFile) +FileChunk::readIdxHeader(FastOS_FileInterface &idxFile, uint32_t &docIdLimit) { int64_t fileSize = idxFile.GetSize(); uint32_t hl = GenericHeader::getMinSize(); @@ -407,9 +413,29 @@ FileChunk::readIdxHeader(FastOS_FileInterface &idxFile) if (idxHeaderLen == 0u) { throw SummaryException("bad file header", idxFile, VESPA_STRLOC); } + GenericHeader::MMapReader reader(static_cast<const char *> (idxFile.MemoryMapPtr(0)), idxHeaderLen); + GenericHeader header; + header.read(reader); + docIdLimit = readDocIdLimit(header); return idxHeaderLen; } +uint32_t +FileChunk::readDocIdLimit(vespalib::GenericHeader &header) +{ + if (header.hasTag(DOC_ID_LIMIT_KEY)) { + return header.getTag(DOC_ID_LIMIT_KEY).asInteger(); + } else { + return std::numeric_limits<uint32_t>::max(); + } +} + +void +FileChunk::writeDocIdLimit(vespalib::GenericHeader &header, uint32_t docIdLimit) +{ + header.putTag(vespalib::GenericHeader::Tag(DOC_ID_LIMIT_KEY, docIdLimit)); +} + void FileChunk::verify(bool reportOnly) const { @@ -482,7 +508,8 @@ FileChunk::isIdxFileEmpty(const vespalib::string & name) if (idxFile.OpenReadOnly()) { if (idxFile.IsMemoryMapped()) { int64_t fileSize = idxFile.getSize(); - int64_t idxHeaderLen = FileChunk::readIdxHeader(idxFile); + uint32_t docIdLimit = std::numeric_limits<uint32_t>::max(); + int64_t idxHeaderLen = FileChunk::readIdxHeader(idxFile, docIdLimit); return fileSize <= idxHeaderLen; } else if ( idxFile.getSize() == 0u) { return true; @@ -512,9 +539,10 @@ FileChunk::getStats() const uint64_t diskBloat = getDiskBloat(); double bucketSpread = getBucketSpread(); uint64_t serialNum = getLastPersistedSerialNum(); + uint32_t docIdLimit = getDocIdLimit(); uint64_t nameId = getNameId().getId(); return DataStoreFileChunkStats(diskFootprint, diskBloat, bucketSpread, - serialNum, serialNum, nameId); + serialNum, serialNum, docIdLimit, nameId); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 9591d13ef94..7243739bbb0 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -15,7 +15,10 @@ class FastOS_FileInterface; -namespace vespalib { class DataBuffer; } +namespace vespalib { + class DataBuffer; + class GenericHeader; +} namespace search { @@ -101,10 +104,11 @@ public: typedef vespalib::hash_map<uint32_t, std::unique_ptr<vespalib::DataBuffer>> LidBufferMap; typedef std::unique_ptr<FileChunk> UP; typedef uint32_t SubChunkId; - FileChunk(FileId fileId, NameId nameId, const vespalib::string & baseName, const TuneFileSummary & tune, const IBucketizer * bucketizer, bool skipCrcOnRead); + FileChunk(FileId fileId, NameId nameId, const vespalib::string &baseName, const TuneFileSummary &tune, + const IBucketizer *bucketizer, bool skipCrcOnRead); virtual ~FileChunk(); - virtual size_t updateLidMap(const LockGuard & guard, ISetLid & lidMap, uint64_t serialNum); + virtual size_t updateLidMap(const LockGuard &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit); virtual ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const; virtual void read(LidInfoWithLidV::const_iterator begin, size_t count, IBufferVisitor & visitor) const; void remove(uint32_t lid, uint32_t size); @@ -149,7 +153,10 @@ public: FileId getFileId() const { return _fileId; } NameId getNameId() const { return _nameId; } size_t getBloatCount() const { return _erasedCount; } + size_t getAddedBytes() const { return _addedBytes; } + size_t getErasedBytes() const { return _erasedBytes; } uint64_t getLastPersistedSerialNum() const; + uint32_t getDocIdLimit() const { return _docIdLimit; } virtual fastos::TimeStamp getModificationTime() const; virtual bool frozen() const { return true; } const vespalib::string & getName() const { return _name; } @@ -181,7 +188,7 @@ public: /** * Read header and return number of bytes it consist of. */ - static uint64_t readIdxHeader(FastOS_FileInterface &idxFile); + static uint64_t readIdxHeader(FastOS_FileInterface &idxFile, uint32_t &docIdLimit); static uint64_t readDataHeader(FileRandRead &idxFile); static bool isIdxFileEmpty(const vespalib::string & name); static void eraseIdxFile(const vespalib::string & name); @@ -224,6 +231,8 @@ protected: void setNumUniqueBuckets(size_t numUniqueBuckets) { _numUniqueBuckets = numUniqueBuckets; } ssize_t read(uint32_t lid, SubChunkId chunkId, const ChunkInfo & chunkInfo, vespalib::DataBuffer & buffer) const; void read(LidInfoWithLidV::const_iterator begin, size_t count, ChunkInfo ci, IBufferVisitor & visitor) const; + static uint32_t readDocIdLimit(vespalib::GenericHeader &header); + static void writeDocIdLimit(vespalib::GenericHeader &header, uint32_t docIdLimit); typedef vespalib::Array<ChunkInfo> ChunkInfoVector; const IBucketizer * _bucketizer; @@ -235,6 +244,7 @@ protected: uint32_t _dataHeaderLen; uint32_t _idxHeaderLen; uint64_t _lastPersistedSerialNum; + uint32_t _docIdLimit; // Limit when the file was created. Stored in idx file header. fastos::TimeStamp _modificationTime; }; diff --git a/searchlib/src/vespa/searchlib/docstore/idatastore.cpp b/searchlib/src/vespa/searchlib/docstore/idatastore.cpp index e0e788968c1..b6b97d95666 100644 --- a/searchlib/src/vespa/searchlib/docstore/idatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/idatastore.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "idatastore.h" namespace search { IDataStore::IDataStore(const vespalib::string& dirName) : - _nextId(0), + _docIdLimit(0), _dirName(dirName) { } @@ -15,5 +14,4 @@ IDataStore::~IDataStore() { } - } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/idatastore.h b/searchlib/src/vespa/searchlib/docstore/idatastore.h index 9351a0f7802..5ae0eeeb9a1 100644 --- a/searchlib/src/vespa/searchlib/docstore/idatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/idatastore.h @@ -175,9 +175,8 @@ public: /** * Get the number of entries (including removed IDs * or gaps in the local ID sequence) in the data store. - * @return The next local ID expected to be used */ - uint64_t nextId() const { return _nextId; } + uint32_t getDocIdLimit() const { return _docIdLimit; } /** * Returns the name of the base directory where the data file is stored. @@ -185,10 +184,17 @@ public: const vespalib::string & getBaseDir() const { return _dirName; } protected: - void setNextId(uint64_t id) { _nextId = id; } + void setDocIdLimit(uint32_t docIdLimit) { + _docIdLimit = docIdLimit; + } + void updateDocIdLimit(uint32_t docIdLimit) { + if (docIdLimit > _docIdLimit) { + setDocIdLimit(docIdLimit); + } + } private: - uint64_t _nextId; + uint32_t _docIdLimit; vespalib::string _dirName; }; diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp index 1042e0dcc12..91c3fce3161 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "idocumentstore.h" namespace search { -IDocumentStore::IDocumentStore(void) +IDocumentStore::IDocumentStore() { } diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h index 0d0e18a0d82..b46e858cce6 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h @@ -64,7 +64,7 @@ public: * @param docMan The document type manager to use when deserializing. * @param baseDir The path to a directory where the implementaion specific files will reside. **/ - IDocumentStore(void); + IDocumentStore(); virtual ~IDocumentStore(); /** @@ -122,9 +122,8 @@ public: /** * Get the number of entries (including removed IDs * or gaps in the local ID sequence) in the document store. - * @return The next local ID expected to be used. */ - virtual uint64_t nextId() const = 0; + virtual uint32_t getDocIdLimit() const = 0; /** * Calculate memory used by this instance. During flush() actual diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index b27c39ba660..fa55ea6d4b2 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -53,14 +53,16 @@ LogDataStore::LogDataStore(vespalib::ThreadExecutor &executor, _executor(executor), _initFlushSyncToken(0), _tlSyncer(tlSyncer), - _bucketizer(bucketizer) + _bucketizer(bucketizer), + _currentlyCompacting(), + _compactLidSpaceGeneration() { // Reserve space for 1TB summary in order to avoid locking. _fileChunks.reserve(LidInfo::getFileIdLimit()); _holdFileChunks.resize(LidInfo::getFileIdLimit()); preload(); - updateLidMap(); + updateLidMap(getLastFileChunkDocIdLimit()); updateSerialNum(); } @@ -86,13 +88,16 @@ LogDataStore::~LogDataStore() } void -LogDataStore::updateLidMap() +LogDataStore::updateLidMap(uint32_t lastFileChunkDocIdLimit) { uint64_t lastSerialNum(0); LockGuard guard(_updateLock); - for (FileChunk::UP & fc : _fileChunks) { - fc->updateLidMap(guard, *this, lastSerialNum); - lastSerialNum = fc->getLastPersistedSerialNum(); + for (size_t i = 0; i < _fileChunks.size(); ++i) { + FileChunk::UP &chunk = _fileChunks[i]; + bool lastChunk = ((i + 1) == _fileChunks.size()); + uint32_t docIdLimit = lastChunk ? std::numeric_limits<uint32_t>::max() : lastFileChunkDocIdLimit; + chunk->updateLidMap(guard, *this, lastSerialNum, docIdLimit); + lastSerialNum = chunk->getLastPersistedSerialNum(); } } @@ -102,7 +107,7 @@ LogDataStore::read(const LidVector & lids, IBufferVisitor & visitor) const LidInfoWithLidV orderedLids; GenerationHandler::Guard guard(_genHandler.takeGuard()); for (uint32_t lid : lids) { - if (lid < _lidInfo.size()) { + if (lid < getDocIdLimit()) { LidInfo li = _lidInfo[lid]; if (!li.empty() && li.valid()) { orderedLids.emplace_back(li, lid); @@ -131,7 +136,7 @@ ssize_t LogDataStore::read(uint32_t lid, vespalib::DataBuffer& buffer) const { ssize_t sz(0); - if (lid < _lidInfo.size()) { + if (lid < getDocIdLimit()) { LidInfo li(0); { GenerationHandler::Guard guard(_genHandler.takeGuard()); @@ -243,7 +248,7 @@ void LogDataStore::remove(uint64_t serialNum, uint32_t lid) { LockGuard guard(_updateLock); - if (lid < _lidInfo.size()) { + if (lid < getDocIdLimit()) { LidInfo lm = _lidInfo[lid]; if (lm.valid()) { _fileChunks[lm.getFileId()]->remove(lid, lm.size()); @@ -557,7 +562,7 @@ LogDataStore::getDiskFootprint() const size_t -LogDataStore::getDiskHeaderFootprint(void) const +LogDataStore::getDiskHeaderFootprint() const { LockGuard guard(_updateLock); size_t sz(0); @@ -619,8 +624,10 @@ LogDataStore::createWritableFile(FileId fileId, SerialNum serialNum, NameId name return FileChunk::UP(); } } + uint32_t docIdLimit = (getDocIdLimit() != 0) ? getDocIdLimit() : std::numeric_limits<uint32_t>::max(); FileChunk::UP file(new WriteableFileChunk(_executor, fileId, nameId, getBaseDir(), - serialNum, _config.getFileConfig(), _tune, _fileHeaderContext, + serialNum, docIdLimit, + _config.getFileConfig(), _tune, _fileHeaderContext, _bucketizer.get(), _config.crcOnReadDisabled())); file->enableRead(); return file; @@ -768,6 +775,15 @@ LogDataStore::preload() _prevActive = _active.prev(); } +uint32_t +LogDataStore::getLastFileChunkDocIdLimit() +{ + if (!_fileChunks.empty()) { + return _fileChunks.back()->getDocIdLimit(); + } + return std::numeric_limits<uint32_t>::max(); +} + LogDataStore::NameIdSet LogDataStore::eraseEmptyIdxFiles(const NameIdSet &partList) { @@ -859,27 +875,33 @@ LogDataStore::scanDir(const vespalib::string &dir, const vespalib::string &suffi } void -LogDataStore::setLid(const LockGuard & guard, uint32_t lid, const LidInfo & meta) +LogDataStore::setLid(const LockGuard &guard, uint32_t lid, const LidInfo &meta) { (void) guard; if (lid < _lidInfo.size()) { _genHandler.updateFirstUsedGeneration(); _lidInfo.removeOldGenerations(_genHandler.getFirstUsedGeneration()); - const LidInfo & prev = _lidInfo[lid]; + const LidInfo &prev = _lidInfo[lid]; if (prev.valid()) { _fileChunks[prev.getFileId()]->remove(lid, prev.size()); } } else { _lidInfo.ensure_size(lid+1, LidInfo()); - _lidInfo.setGeneration(_genHandler.getNextGeneration()); - _genHandler.incGeneration(); - _genHandler.updateFirstUsedGeneration(); - _lidInfo.removeOldGenerations(_genHandler.getFirstUsedGeneration()); - setNextId(_lidInfo.size()); + incGeneration(); } + updateDocIdLimit(lid + 1); _lidInfo[lid] = meta; } +void +LogDataStore::incGeneration() +{ + _lidInfo.setGeneration(_genHandler.getNextGeneration()); + _genHandler.incGeneration(); + _genHandler.updateFirstUsedGeneration(); + _lidInfo.removeOldGenerations(_genHandler.getFirstUsedGeneration()); +} + size_t LogDataStore::computeNumberOfSignificantBucketIdBits(const IBucketizer & bucketizer, FileId fileId) const { @@ -889,7 +911,7 @@ LogDataStore::computeNumberOfSignificantBucketIdBits(const IBucketizer & bucketi timer.before(); auto bucketizerGuard = bucketizer.getGuard(); GenerationHandler::Guard lidGuard(_genHandler.takeGuard()); - for (size_t i(0), m(_lidInfo.size()); i < m; i++) { + for (size_t i(0), m(getDocIdLimit()); i < m; i++) { LidInfo lid(_lidInfo[i]); if (lid.valid() && (lid.getFileId() == fileId.getId())) { BucketId bucketId = bucketizer.getBucketOf(bucketizerGuard, i); @@ -1065,8 +1087,9 @@ LogDataStore::getStorageStats() const // Note: Naming consistency issue SerialNum lastSerialNum = tentativeLastSyncToken(); SerialNum lastFlushedSerialNum = lastSyncToken(); + uint32_t docIdLimit = getDocIdLimit(); return DataStoreStorageStats(diskFootprint, diskBloat, maxBucketSpread, - lastSerialNum, lastFlushedSerialNum); + lastSerialNum, lastFlushedSerialNum, docIdLimit); } MemoryUsage @@ -1102,18 +1125,49 @@ LogDataStore::getFileChunkStats() const void LogDataStore::compactLidSpace(uint32_t wantedDocLidLimit) { - (void) wantedDocLidLimit; + LockGuard guard(_updateLock); + assert(wantedDocLidLimit <= getDocIdLimit()); + for (size_t i = wantedDocLidLimit; i < _lidInfo.size(); ++i) { + _lidInfo[i] = LidInfo(); + } + setDocIdLimit(wantedDocLidLimit); + _compactLidSpaceGeneration = _genHandler.getCurrentGeneration(); + incGeneration(); } bool LogDataStore::canShrinkLidSpace() const { - return false; + LockGuard guard(_updateLock); + return canShrinkLidSpace(guard); +} + +bool +LogDataStore::canShrinkLidSpace(const vespalib::LockGuard &) const +{ + return getDocIdLimit() < _lidInfo.size() && + _compactLidSpaceGeneration < _genHandler.getFirstUsedGeneration(); +} + +size_t +LogDataStore::getEstimatedShrinkLidSpaceGain() const +{ + LockGuard guard(_updateLock); + if (!canShrinkLidSpace(guard)) { + return 0; + } + return (_lidInfo.size() - getDocIdLimit()) * sizeof(uint64_t); } void LogDataStore::shrinkLidSpace() { + LockGuard guard(_updateLock); + if (!canShrinkLidSpace(guard)) { + return; + } + _lidInfo.shrink(getDocIdLimit()); + incGeneration(); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 6688484887c..d5d22123fac 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -43,7 +43,8 @@ public: _numThreads(8), _skipCrcOnRead(false), _compactToActiveFile(true), - _compactCompression(CompressionConfig::LZ4) + _compactCompression(CompressionConfig::LZ4), + _fileConfig() { } Config(size_t maxFileSize, @@ -177,7 +178,11 @@ public: // Implements IGetLid API LidInfo getLid(Guard & guard, uint32_t lid) const override { (void) guard; - return _lidInfo[lid]; + if (lid < getDocIdLimit()) { + return _lidInfo[lid]; + } else { + return LidInfo(); + } } FileId getActiveFileId(const vespalib::LockGuard & guard) const { assert(guard.locks(_updateLock)); @@ -197,6 +202,7 @@ public: */ virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; virtual bool canShrinkLidSpace() const override; + virtual size_t getEstimatedShrinkLidSpaceGain() const override; virtual void shrinkLidSpace() override; private: @@ -216,13 +222,14 @@ private: typedef attribute::RcuVector<uint64_t> LidInfoVector; typedef std::vector<FileChunk::UP> FileChunkVector; - void updateLidMap(); + void updateLidMap(uint32_t lastFileChunkDocIdLimit); void preload(); + uint32_t getLastFileChunkDocIdLimit(); void verifyModificationTime(const NameIdSet & partList); void eraseDanglingDatFiles(const NameIdSet &partList, const NameIdSet &datPartList); NameIdSet eraseEmptyIdxFiles(const NameIdSet &partList); - void internalFlushAll(void); + void internalFlushAll(); NameIdSet scanDir(const vespalib::string &dir, const vespalib::string &suffix); FileId allocateFileId(const LockGuard & guard); @@ -298,6 +305,8 @@ private: } bool shouldCompactToActiveFile(size_t compactedSize) const; std::pair<bool, FileId> findNextToCompact(); + void incGeneration(); + bool canShrinkLidSpace(const vespalib::LockGuard &guard) const; typedef std::vector<FileId> FileIdxVector; Config _config; @@ -316,6 +325,7 @@ private: transactionlog::SyncProxy &_tlSyncer; IBucketizer::SP _bucketizer; NameIdSet _currentlyCompacting; + uint64_t _compactLidSpaceGeneration; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp index 348d40b37f5..1be982c7310 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "logdocumentstore.h" -namespace search -{ +namespace search { using vespalib::nbostream; using common::FileHeaderContext; diff --git a/searchlib/src/vespa/searchlib/docstore/randread.h b/searchlib/src/vespa/searchlib/docstore/randread.h index e623cda32f1..00820594ad3 100644 --- a/searchlib/src/vespa/searchlib/docstore/randread.h +++ b/searchlib/src/vespa/searchlib/docstore/randread.h @@ -18,7 +18,7 @@ public: typedef std::shared_ptr<FastOS_FileInterface> FSP; virtual ~FileRandRead() { } virtual FSP read(size_t offset, vespalib::DataBuffer & buffer, size_t sz) = 0; - virtual int64_t getSize(void) = 0; + virtual int64_t getSize() = 0; }; } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index c424cb61d8c..8fac288a23a 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -4,9 +4,10 @@ #include "ibucketizer.h" #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/document/util/compressor.h> -namespace search { -namespace docstore { +namespace search::docstore { using vespalib::ConstBufferRef; using vespalib::LockGuard; @@ -241,5 +242,3 @@ VisitCache::Cache::onRemove(const K & key) { } } -} - diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 76519daa636..6e5887349ee 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -6,11 +6,12 @@ #include "cachestats.h" #include <vespa/vespalib/stllike/cache.h> #include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/document/util/bytebuffer.h> -namespace search { -namespace docstore { +namespace search::docstore { /** * Represents a unique set of keys that together acts as a key in a map. @@ -117,7 +118,7 @@ private: bool read(const KeySet &key, CompressedBlobSet &blobs) const; void write(const KeySet &, const CompressedBlobSet &) { } void erase(const KeySet &) { } - const document::CompressionConfig &getCompression(void) const { return _compression; } + const document::CompressionConfig &getCompression() const { return _compression; } private: IDataStore &_backingStore; const document::CompressionConfig _compression; @@ -160,4 +161,3 @@ private: }; } -} diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index e5849703646..5cb11ede09c 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -46,13 +46,13 @@ class PendingChunk public: typedef std::shared_ptr<PendingChunk> SP; PendingChunk(uint64_t lastSerial, uint64_t dataOffset, uint32_t dataLen); - ~PendingChunk(void); - vespalib::nbostream & getSerializedIdx(void) { return _idx; } - const vespalib::nbostream & getSerializedIdx(void) const { return _idx; } - uint64_t getDataOffset(void) const { return _dataOffset; } - uint32_t getDataLen(void) const { return _dataLen; } - uint32_t getIdxLen(void) const { return _idx.size(); } - uint64_t getLastSerial(void) const { return _lastSerial; } + ~PendingChunk(); + vespalib::nbostream & getSerializedIdx() { return _idx; } + const vespalib::nbostream & getSerializedIdx() const { return _idx; } + uint64_t getDataOffset() const { return _dataOffset; } + uint32_t getDataLen() const { return _dataLen; } + uint32_t getIdxLen() const { return _idx.size(); } + uint64_t getLastSerial() const { return _lastSerial; } }; class ProcessedChunk @@ -80,6 +80,7 @@ WriteableFileChunk(vespalib::ThreadExecutor &executor, FileId fileId, NameId nameId, const vespalib::string &baseName, SerialNum initialSerialNum, + uint32_t docIdLimit, const Config &config, const TuneFileSummary &tune, const FileHeaderContext &fileHeaderContext, @@ -109,6 +110,7 @@ WriteableFileChunk(vespalib::ThreadExecutor &executor, _executor(executor), _bucketMap(bucketizer) { + _docIdLimit = docIdLimit; if (tune._write.getWantDirectIO()) { _dataFile.EnableDirectIO(); } @@ -131,7 +133,7 @@ WriteableFileChunk(vespalib::ThreadExecutor &executor, if (_idxFile.OpenReadWrite()) { readIdxHeader(); if (_idxHeaderLen == 0) { - _idxHeaderLen = writeIdxHeader(fileHeaderContext, _idxFile); + _idxHeaderLen = writeIdxHeader(fileHeaderContext, _docIdLimit, _idxFile); } _idxFile.SetPosition(_idxFile.GetSize()); } else { @@ -168,9 +170,9 @@ WriteableFileChunk::~WriteableFileChunk() } size_t -WriteableFileChunk::updateLidMap(const LockGuard & guard, ISetLid & ds, uint64_t serialNum) +WriteableFileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit) { - size_t sz = FileChunk::updateLidMap(guard, ds, serialNum); + size_t sz = FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit); _nextChunkId = _chunkInfo.size(); _active.reset( new Chunk(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()))); _serialNum = getLastPersistedSerialNum(); @@ -181,10 +183,8 @@ WriteableFileChunk::updateLidMap(const LockGuard & guard, ISetLid & ds, uint64_t } void -WriteableFileChunk::restart(const MonitorGuard & guard, uint32_t nextChunkId) +WriteableFileChunk::restart(uint32_t nextChunkId) { - (void) guard; - _writeTaskIsRunning = true; _executor.execute(makeTask(makeClosure(this, &WriteableFileChunk::fileWriter, nextChunkId))); } @@ -296,9 +296,14 @@ WriteableFileChunk::enque(ProcessedChunk::UP tmp) MonitorGuard guard(_writeMonitor); _writeQ.push_back(std::move(tmp)); if (_writeTaskIsRunning == false) { - restart(guard, _firstChunkIdToBeWritten); + _writeTaskIsRunning = true; + uint32_t nextChunkId = _firstChunkIdToBeWritten; + guard.signal(); + guard.unlock(); + restart(nextChunkId); + } else { + guard.signal(); } - guard.signal(); } namespace { @@ -505,8 +510,8 @@ WriteableFileChunk::fileWriter(const uint32_t firstChunkId) LOG(debug, "Stopping the filewriter with startchunkid = %d and ending chunkid = %d done=%d", firstChunkId, nextChunkId, done); + MonitorGuard guard(_writeMonitor); if (done) { - MonitorGuard guard(_writeMonitor); assert(_writeQ.empty()); assert(_chunkMap.empty()); for (const ChunkInfo & cm : _chunkInfo) { @@ -516,12 +521,13 @@ WriteableFileChunk::fileWriter(const uint32_t firstChunkId) _writeTaskIsRunning = false; guard.broadcast(); } else { - MonitorGuard guard(_writeMonitor); if (_writeQ.empty()) { _firstChunkIdToBeWritten = nextChunkId; _writeTaskIsRunning = false; } else { - restart(guard, nextChunkId); + _writeTaskIsRunning = true; + guard.unlock(); + restart(nextChunkId); } } } @@ -712,7 +718,7 @@ WriteableFileChunk::append(uint64_t serialNum, void -WriteableFileChunk::readDataHeader(void) +WriteableFileChunk::readDataHeader() { int64_t fSize(_dataFile.GetSize()); try { @@ -744,13 +750,14 @@ WriteableFileChunk::readDataHeader(void) void -WriteableFileChunk::readIdxHeader(void) +WriteableFileChunk::readIdxHeader() { int64_t fSize(_idxFile.GetSize()); try { FileHeader h; _idxHeaderLen = h.readFile(_idxFile); _idxFile.SetPosition(_idxHeaderLen); + _docIdLimit = readDocIdLimit(h); } catch (IllegalHeaderException &e) { _idxFile.SetPosition(0); try { @@ -790,7 +797,7 @@ WriteableFileChunk::writeDataHeader(const FileHeaderContext &fileHeaderContext) uint64_t -WriteableFileChunk::writeIdxHeader(const FileHeaderContext &fileHeaderContext, FastOS_FileInterface & file) +WriteableFileChunk::writeIdxHeader(const FileHeaderContext &fileHeaderContext, uint32_t docIdLimit, FastOS_FileInterface &file) { typedef FileHeader::Tag Tag; FileHeader h; @@ -799,6 +806,7 @@ WriteableFileChunk::writeIdxHeader(const FileHeaderContext &fileHeaderContext, F assert(file.GetPosition() == 0); fileHeaderContext.addTags(h, file.GetFileName()); h.putTag(Tag("desc", "Log data store chunk index")); + writeDocIdLimit(h, docIdLimit); return h.writeFile(file); } @@ -903,7 +911,7 @@ WriteableFileChunk::getStats() const DataStoreFileChunkStats stats = FileChunk::getStats(); uint64_t serialNum = getSerialNum(); return DataStoreFileChunkStats(stats.diskUsage(), stats.diskBloat(), stats.maxBucketSpread(), - serialNum, stats.lastFlushedSerialNum(), stats.nameId()); + serialNum, stats.lastFlushedSerialNum(), stats.docIdLimit(), stats.nameId()); }; PendingChunk::PendingChunk(uint64_t lastSerial, uint64_t dataOffset, uint32_t dataLen) diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 91031187976..37bf102a868 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -45,6 +45,7 @@ public: FileId fileId, NameId nameId, const vespalib::string & baseName, uint64_t initialSerialNum, + uint32_t docIdLimit, const Config & config, const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, @@ -60,18 +61,18 @@ public: uint64_t getSerialNum() const { return _serialNum; } void setSerialNum(uint64_t serialNum) { _serialNum = std::max(_serialNum, serialNum); } - virtual fastos::TimeStamp getModificationTime() const override; + fastos::TimeStamp getModificationTime() const override; void freeze(); size_t getDiskFootprint() const override; size_t getMemoryFootprint() const override; size_t getMemoryMetaFootprint() const override; - virtual MemoryUsage getMemoryUsage() const override; - size_t updateLidMap(const LockGuard & guard, ISetLid & lidMap, uint64_t serialNum) override; + MemoryUsage getMemoryUsage() const override; + size_t updateLidMap(const LockGuard &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit) override; void waitForDiskToCatchUpToNow() const; void flushPendingChunks(uint64_t serialNum); - virtual DataStoreFileChunkStats getStats() const override; + DataStoreFileChunkStats getStats() const override; - static uint64_t writeIdxHeader(const common::FileHeaderContext &fileHeaderContext, FastOS_FileInterface & file); + static uint64_t writeIdxHeader(const common::FileHeaderContext &fileHeaderContext, uint32_t docIdLimit, FastOS_FileInterface &file); private: using ProcessedChunkUP = std::unique_ptr<ProcessedChunk>; typedef std::map<uint32_t, ProcessedChunkUP > ProcessedChunkMap; @@ -85,17 +86,17 @@ private: void internalFlush(uint32_t, uint64_t serialNum); void enque(ProcessedChunkUP); int32_t flushLastIfNonEmpty(bool force); - void restart(const vespalib::MonitorGuard & guard, uint32_t nextChunkId); + // _writeMonitor should not be held when calling restart + void restart(uint32_t nextChunkId); ProcessedChunkQ drainQ(); - void readDataHeader(void); - void readIdxHeader(void); + void readDataHeader(); + void readIdxHeader(); void writeDataHeader(const common::FileHeaderContext &fileHeaderContext); bool needFlushPendingChunks(uint64_t serialNum, uint64_t datFileLen); bool needFlushPendingChunks(const vespalib::MonitorGuard & guard, uint64_t serialNum, uint64_t datFileLen); fastos::TimeStamp unconditionallyFlushPendingChunks(const vespalib::LockGuard & flushGuard, uint64_t serialNum, uint64_t datFileLen); static void insertChunks(ProcessedChunkMap & orderedChunks, ProcessedChunkQ & newChunks, const uint32_t nextChunkId); static ProcessedChunkQ fetchNextChain(ProcessedChunkMap & orderedChunks, const uint32_t firstChunkId); - size_t computeDataLen(const ProcessedChunk & tmp, const Chunk & active); ChunkMeta computeChunkMeta(const vespalib::LockGuard & guard, const vespalib::GenerationHandler::Guard & bucketizerGuard, size_t offset, const ProcessedChunk & tmp, const Chunk & active); @@ -136,4 +137,3 @@ private: }; } // namespace search - diff --git a/searchlib/src/vespa/searchlib/engine/create-class-cpp.sh b/searchlib/src/vespa/searchlib/engine/create-class-cpp.sh index 1f638bf0cb6..8c17349d2b0 100755 --- a/searchlib/src/vespa/searchlib/engine/create-class-cpp.sh +++ b/searchlib/src/vespa/searchlib/engine/create-class-cpp.sh @@ -8,13 +8,9 @@ name=`echo $class | tr 'A-Z' 'a-z'` cat <<EOF // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP(".engine.$name"); -#include <vespa/fastos/fastos.h> #include "$name.h" -namespace search { -namespace engine { +namespace search::engine { $class::$class() { @@ -24,6 +20,5 @@ $class::~$class() { } -} // namespace engine -} // namespace search +} // namespace search::engine EOF diff --git a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp index 7d443219f29..eaa96587d6d 100644 --- a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp @@ -4,7 +4,7 @@ #include "integerresultnode.h" #include "stringresultnode.h" #include <vespa/searchcommon/attribute/iattributecontext.h> - +#include <vespa/vespalib/util/stringfmt.h> namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/expression/arrayoperationnode.cpp b/searchlib/src/vespa/searchlib/expression/arrayoperationnode.cpp index eccc1dfe02f..ce6259177b6 100644 --- a/searchlib/src/vespa/searchlib/expression/arrayoperationnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/arrayoperationnode.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "arrayoperationnode.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/vespa/searchlib/expression/bucketresultnode.cpp b/searchlib/src/vespa/searchlib/expression/bucketresultnode.cpp index e82cef0563a..54961c4fa14 100644 --- a/searchlib/src/vespa/searchlib/expression/bucketresultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/bucketresultnode.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bucketresultnode.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/expression/catserializer.cpp b/searchlib/src/vespa/searchlib/expression/catserializer.cpp index 7761bd83f9c..7df3e9f8297 100644 --- a/searchlib/src/vespa/searchlib/expression/catserializer.cpp +++ b/searchlib/src/vespa/searchlib/expression/catserializer.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "catserializer.h" #include "rawresultnode.h" #include "resultvector.h" diff --git a/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp index 506a8983994..d0dd57e9218 100644 --- a/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/expression/debugwaitfunctionnode.h> +#include "debugwaitfunctionnode.h" +#include <vespa/fastos/time.h> +#include <thread> namespace search { namespace expression { @@ -40,7 +41,7 @@ DebugWaitFunctionNode::onExecute() const ; } else { int rem = (int)(millis - time.MilliSecsToNow()); - FastOS_Thread::Sleep(rem); + std::this_thread::sleep_for(std::chrono::milliseconds(rem)); } } getArg().execute(); diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index 3a5375b672b..d9e4f6a3253 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -3,6 +3,7 @@ #include "getdocidnamespacespecificfunctionnode.h" #include "getymumchecksumfunctionnode.h" #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h index a6c027beb22..2a202fe562e 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h @@ -4,6 +4,7 @@ #include "documentaccessornode.h" #include "resultnode.h" #include "resultvector.h" +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace search { namespace expression { @@ -38,11 +39,9 @@ public: DocumentFieldNode & operator = (const DocumentFieldNode & rhs); const vespalib::string & getFieldName() const override { return _fieldName; } private: - class Handler : public document::FieldValue::IteratorHandler { + class Handler : public document::fieldvalue::IteratorHandler { public: virtual void reset() = 0; - protected: - typedef document::FieldValue::IteratorHandler::Content Content; private: void onCollectionStart(const Content & c) override; void onStructStart(const Content & c) override; diff --git a/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp index c4f2f725271..a08f9653da0 100644 --- a/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fixedwidthbucketfunctionnode.h" #include "integerresultnode.h" #include "floatresultnode.h" diff --git a/searchlib/src/vespa/searchlib/expression/floatbucketresultnode.cpp b/searchlib/src/vespa/searchlib/expression/floatbucketresultnode.cpp index 950c5f5a259..e3d0d2da2d4 100644 --- a/searchlib/src/vespa/searchlib/expression/floatbucketresultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/floatbucketresultnode.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "floatbucketresultnode.h" #include <vespa/vespalib/objects/visit.h> #include <cmath> diff --git a/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.cpp b/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.cpp index f821026a679..d1e709b426f 100644 --- a/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/integerbucketresultnode.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "integerbucketresultnode.h" #include <vespa/vespalib/objects/visit.h> diff --git a/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp index d135be103e6..84f146c9014 100644 --- a/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/expression/interpolatedlookupfunctionnode.h> -#include <vespa/searchlib/expression/floatresultnode.h> +#include "interpolatedlookupfunctionnode.h" +#include "floatresultnode.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/converters.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp index 14cdb627f58..36649c461b4 100644 --- a/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/expression/mathfunctionnode.h> -#include <vespa/searchlib/expression/floatresultnode.h> +#include "mathfunctionnode.h" +#include "floatresultnode.h" #include <cmath> namespace search { diff --git a/searchlib/src/vespa/searchlib/expression/rawbucketresultnode.cpp b/searchlib/src/vespa/searchlib/expression/rawbucketresultnode.cpp index 344903e5820..f65d1e6d9b6 100644 --- a/searchlib/src/vespa/searchlib/expression/rawbucketresultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/rawbucketresultnode.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "rawbucketresultnode.h" #include <vespa/vespalib/objects/visit.hpp> diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.cpp b/searchlib/src/vespa/searchlib/expression/resultnode.cpp index 387b05e7add..bd8a5ff03d5 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnode.cpp @@ -1,11 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/expression/resultnode.h> -#include <stdexcept> + +#include "resultnode.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/exception.h> namespace search { namespace expression { +int64_t +ResultNode::onGetEnum(size_t index) const { + (void) index; + throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented"); +} + uint64_t ResultNode::radixAsc(const void * buf) const { (void) buf; @@ -66,6 +73,11 @@ size_t ResultNode::getRawByteSize() const throw std::runtime_error("ResultNode::getRawByteSize() const must be overloaded by '" + vespalib::string(getClass().name()) + "'."); } +const BucketResultNode& +ResultNode::getNullBucket() const { + throw std::runtime_error(vespalib::make_string("No null bucket defined for this type")); +} + } } diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.h b/searchlib/src/vespa/searchlib/expression/resultnode.h index 6343e90dc6b..26849d27ed3 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnode.h +++ b/searchlib/src/vespa/searchlib/expression/resultnode.h @@ -3,7 +3,6 @@ #include "expressionnode.h" #include "serializer.h" -#include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/buffer.h> namespace search { @@ -49,10 +48,7 @@ public: private: virtual int64_t onGetInteger(size_t index) const = 0; - virtual int64_t onGetEnum(size_t index) const { - (void) index; - throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented"); - } + virtual int64_t onGetEnum(size_t index) const; virtual double onGetFloat(size_t index) const = 0; virtual ConstBufferRef onGetString(size_t index, BufferRef buf) const = 0; @@ -124,9 +120,8 @@ public: virtual ResultDeserializer & onDeserializeResult(ResultDeserializer & is); virtual size_t getRawByteSize() const; virtual bool isMultiValue() const { return false; } - virtual const BucketResultNode& getNullBucket() const { throw std::runtime_error(vespalib::make_string("No null bucket defined for this type")); } + virtual const BucketResultNode& getNullBucket() const; }; } } - diff --git a/searchlib/src/vespa/searchlib/expression/resultvector.cpp b/searchlib/src/vespa/searchlib/expression/resultvector.cpp index 169b4ea6159..991cff9997c 100644 --- a/searchlib/src/vespa/searchlib/expression/resultvector.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultvector.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/expression/resultvector.h> + +#include "resultvector.h" namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/expression/stringbucketresultnode.cpp b/searchlib/src/vespa/searchlib/expression/stringbucketresultnode.cpp index de7ef05f31c..f53d1bbce07 100644 --- a/searchlib/src/vespa/searchlib/expression/stringbucketresultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/stringbucketresultnode.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "stringbucketresultnode.h" #include <vespa/vespalib/objects/visit.hpp> diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp index 6621f9225cb..d46279bbd30 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.agefeature"); -#include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/matchdata.h> #include "agefeature.h" #include "valuefeature.h" +#include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/searchlib/fef/matchdata.h> using search::attribute::IAttributeVector; diff --git a/searchlib/src/vespa/searchlib/features/array_parser.cpp b/searchlib/src/vespa/searchlib/features/array_parser.cpp index 0faac986033..fba9d04e19b 100644 --- a/searchlib/src/vespa/searchlib/features/array_parser.cpp +++ b/searchlib/src/vespa/searchlib/features/array_parser.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "array_parser.h" + #include <vespa/log/log.h> LOG_SETUP(".features.array_parser"); -#include "array_parser.h" - namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 7622f0765ff..98cf4a35764 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.attributematchfeature"); - -#include <vespa/searchcommon/attribute/attributecontent.h> -#include <vespa/searchlib/fef/featurenamebuilder.h> -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/properties.h> #include "attributematchfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/searchlib/fef/fieldinfo.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchcommon/attribute/attributecontent.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.attributematchfeature"); using namespace search::attribute; using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index eb1a5c646c0..03c4ad1c3f5 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.closenessfeature"); -#include <vespa/searchlib/fef/properties.h> #include "closenessfeature.h" #include "utils.h" +#include <vespa/searchlib/fef/properties.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.closenessfeature"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp index 54fc40b0019..fda0215dff0 100644 --- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.constant_feature"); #include "constant_feature.h" - -#include <vespa/searchlib/fef/featureexecutor.h> #include "valuefeature.h" +#include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/eval/value_cache/constant_value.h> +#include <vespa/log/log.h> +LOG_SETUP(".features.constant_feature"); + using namespace search::fef; namespace search { diff --git a/searchlib/src/vespa/searchlib/features/create-class-cpp.sh b/searchlib/src/vespa/searchlib/features/create-class-cpp.sh index 6ec335ffa3d..abc25ab99d4 100755 --- a/searchlib/src/vespa/searchlib/features/create-class-cpp.sh +++ b/searchlib/src/vespa/searchlib/features/create-class-cpp.sh @@ -6,15 +6,11 @@ guard=`echo $class | tr 'a-z' 'A-Z'` name=`echo $class | tr 'A-Z' 'a-z'` cat <<EOF -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP(".$name"); -#include <vespa/fastos/fastos.h> #include "$name.h" -namespace search { -namespace features { +namespace search::features { $class::$class() { @@ -24,6 +20,5 @@ $class::~$class() { } -} // namespace features -} // namespace search +} EOF diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp index 615fd06c987..19814760fc6 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.debug_wait"); #include "debug_attribute_wait.h" +#include <thread> using search::attribute::IAttributeVector; @@ -58,7 +56,7 @@ DebugAttributeWaitExecutor::execute(uint32_t docId) ; } else { int rem = (int)(millis - time.MilliSecsToNow()); - FastOS_Thread::Sleep(rem); + std::this_thread::sleep_for(std::chrono::milliseconds(rem)); } } outputs().set_number(0, 1.0e-6 * time.MicroSecsToNow()); diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp index 395a4c11d61..b9896b22946 100644 --- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.debug_wait"); #include "debug_wait.h" +#include <thread> namespace search { @@ -42,7 +40,7 @@ DebugWaitExecutor::execute(uint32_t) ; } else { int rem = (int)(millis - time.MilliSecsToNow()); - FastOS_Thread::Sleep(rem); + std::this_thread::sleep_for(std::chrono::milliseconds(rem)); } } outputs().set_number(0, 1.0e-6 * time.MicroSecsToNow()); diff --git a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp index e7515727b23..30ca8af45bc 100644 --- a/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/dense_tensor_attribute_executor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_attribute_executor.h" #include <vespa/searchlib/tensor/dense_tensor_attribute.h> diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index d38e19e93d2..50ad687adc1 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -1,15 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.distancefeature"); -#include <cmath> -#include <limits> -#include <vespa/document/datatype/positiondatatype.h> +#include "distancefeature.h" #include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> -#include "distancefeature.h" +#include <cmath> +#include <limits> + +#include <vespa/log/log.h> +LOG_SETUP(".features.distancefeature"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index da9db13f17c..91878fc948e 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -1,18 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.distancetopathfeature"); +#include "distancetopathfeature.h" +#include "utils.h" +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/document/datatype/positiondatatype.h> +#include <vespa/vespalib/geo/zcurve.h> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <cmath> -#include <vespa/document/datatype/positiondatatype.h> -#include <vespa/searchlib/fef/matchdata.h> -#include <vespa/searchlib/fef/properties.h> #include <sstream> -#include <vespa/vespalib/geo/zcurve.h> -#include "distancetopathfeature.h" -#include "utils.h" + +#include <vespa/log/log.h> +LOG_SETUP(".features.distancetopathfeature"); namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index 0874afcb080..71bb8731c7c 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.elementcompleteness"); #include "element_completeness_feature.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp index 86bd40cb050..1f689ce8dd1 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldinfo"); - +#include "fieldinfofeature.h" +#include "valuefeature.h" +#include "utils.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> @@ -11,9 +10,6 @@ LOG_SETUP(".features.fieldinfo"); #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> #include <sstream> -#include "fieldinfofeature.h" -#include "valuefeature.h" -#include "utils.h" namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp index 413c538ba85..26e11bbb5d3 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldlength"); +#include "fieldlengthfeature.h" +#include "valuefeature.h" +#include "utils.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/vespalib/util/stringfmt.h> -#include "fieldlengthfeature.h" -#include "valuefeature.h" -#include "utils.h" using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp index d613083233f..23847b11fd4 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp @@ -1,15 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldmatch.computer"); -#include <iostream> -#include <cstdlib> -#include <set> +#include "computer.h" #include <vespa/searchlib/features/utils.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> -#include "computer.h" +#include <iostream> +#include <cstdlib> +#include <set> + +#include <vespa/log/log.h> +LOG_SETUP(".features.fieldmatch.computer"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp index d89b447a794..1d7cfa59f17 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldmatch.metrics"); +#include "metrics.h" +#include "computer.h" +#include <vespa/vespalib/util/stringfmt.h> #include <algorithm> #include <cstdlib> #include <cmath> #include <cstdlib> -#include <vespa/vespalib/util/stringfmt.h> -#include "computer.h" -#include "metrics.h" namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h index 6b826f09e57..ed39fe39f95 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> #include <vespa/searchlib/common/feature.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp index 34a23bb3642..9543cf8d174 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "params.h" + #include <vespa/log/log.h> LOG_SETUP(".features.fieldmatch.params"); -#include "params.h" - namespace search { namespace features { namespace fieldmatch { diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h index f3ff7558971..5496ce00ff7 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> #include <vespa/searchlib/common/feature.h> +#include <vector> +#include <cstdint> namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp index 93be549bc1f..ca6a92b9ed3 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldmatch.segmentstart"); -#include <vespa/vespalib/util/stringfmt.h> +#include "segmentstart.h" #include "computer.h" #include "metrics.h" -#include "segmentstart.h" +#include <vespa/vespalib/util/stringfmt.h> namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp index 6a6e1935ad3..705fb3de45d 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldmatch.simplemetrics"); #include "simplemetrics.h" #include <vespa/vespalib/stllike/asciistream.h> diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp index 6c0bf057c43..2342547c16b 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.fieldtermmatchfeature"); +#include "fieldtermmatchfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "fieldtermmatchfeature.h" -#include "utils.h" namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp index daa20d06e5b..a56b0dd65bc 100644 --- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.firstphasefeature"); #include "firstphasefeature.h" - #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp index d6955118ccb..afacee26b26 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.freshnessfeature"); -#include <vespa/searchlib/fef/properties.h> #include "freshnessfeature.h" #include "utils.h" +#include <vespa/searchlib/fef/properties.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.freshnessfeature"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index 14b517a10ca..5088dfdffbd 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.item_raw_score_feature"); #include "item_raw_score_feature.h" #include "valuefeature.h" #include "utils.h" diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp index 0c933261920..357a5e0b106 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp @@ -1,17 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.jarowinklerdistance"); +#include "jarowinklerdistancefeature.h" +#include "utils.h" +#include <limits> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "jarowinklerdistancefeature.h" -#include "utils.h" -#include <limits> namespace search { namespace features { @@ -131,12 +128,9 @@ feature_t JaroWinklerDistanceExecutor::jaroWinklerProximity(const std::vector<search::fef::FieldPositionsIterator> &termPos, uint32_t fieldLen) { feature_t ret = std::min(1.0, std::max(0.0, jaroMeasure(termPos, fieldLen))); - //LOG(debug, "Jaro measure is %f.", ret); if (ret > _config.boostThreshold) { ret += 0.1f * prefixMatch(termPos, fieldLen, _config.prefixSize) * (1 - ret); // less boost close to 1 - //LOG(debug, "Applying Winkler boost."); } - //LOG(debug, "JaroWinkler measure is %f.", ret); return ret; } diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index e62b1a91785..ee8ec3eeca9 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.matchesfeature"); -#include <vespa/searchlib/fef/fieldinfo.h> #include "matchesfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/searchlib/fef/fieldinfo.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp index 780c52e9508..5c8feedc9bb 100644 --- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.matchfeature"); #include "matchfeature.h" - +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> -#include "utils.h" using namespace search::fef; using CollectionType = FieldInfo::CollectionType; diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index 9e2384b08c8..3b717e43397 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.native_dot_product_feature"); #include "native_dot_product_feature.h" #include "utils.h" diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 6d92e8694bf..b10e2e70ff0 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "valuefeature.h" +#include "nativeattributematchfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> -#include "valuefeature.h" -#include "nativeattributematchfeature.h" -#include "utils.h" -LOG_SETUP(".features.nativeattributematchfeature"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index 3a333211e1a..edfc5435fc2 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.nativefieldmatchfeature"); +#include "nativefieldmatchfeature.h" +#include "valuefeature.h" +#include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> -#include "nativefieldmatchfeature.h" -#include "valuefeature.h" -#include "utils.h" using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index ddd4dcec47e..aeb9f0c8f01 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.nativeproximityfeature"); +#include "nativeproximityfeature.h" +#include "valuefeature.h" +#include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> -#include "nativeproximityfeature.h" -#include "valuefeature.h" -#include "utils.h" #include <map> using namespace search::fef; @@ -47,10 +44,6 @@ NativeProximityExecutor::calculateScoreForPair(const TermPair & pair, uint32_t f (a.significance() * a.termData()->getWeight().percent() + b.significance() * b.termData()->getWeight().percent()); feature_t score = (forwardScore + reverseScore) * termPairWeight / param.maxTableSum; - //LOG(debug, "calculateScoreForPair: pair(%u,%u), fieldId(%u), forwardScore(%f), reverseScore(%f), " - //"termPairWeight(%f), maxTableSum(%f), score(%f)", - //fieldId, a.termData()->getUniqueId(), b.termData()->getUniqueId(), forwardScore, reverseScore, - //termPairWeight, _params.maxTableSums[fieldId], score); return score; } diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp index ee812f2173e..924318a4a5a 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.nativerankfeature"); -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/properties.h> #include "nativerankfeature.h" #include "valuefeature.h" #include "utils.h" +#include <vespa/searchlib/fef/properties.h> +#include <sstream> + +#include <vespa/log/log.h> +LOG_SETUP(".features.nativerankfeature"); using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp index cc6614d8c06..5fcddcebd1e 100644 --- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.nowfeature"); +#include "nowfeature.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/properties.h> -#include "nowfeature.h" -#include "valuefeature.h" +#include <vespa/fastos/time.h> namespace search { namespace features { @@ -16,7 +13,6 @@ NowExecutor::NowExecutor(int64_t timestamp) : search::fef::FeatureExecutor(), _timestamp(timestamp) { - // empty } void diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp index b8a69c38b32..62f57efbc2d 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.proximity"); +#include "proximityfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "proximityfeature.h" -#include "utils.h" namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp index f6db8a34534..ab876509e0f 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp @@ -1,15 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.querycompleteness"); +#include "querycompletenessfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/itermdata.h> -#include "querycompletenessfeature.h" -#include "utils.h" #include <limits> +#include <vespa/log/log.h> +LOG_SETUP(".features.querycompleteness"); + namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index 4cb1af346cf..d61e2928165 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -1,23 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.queryfeature"); #include "queryfeature.h" +#include "utils.h" +#include "valuefeature.h" #include <vespa/searchlib/features/constant_tensor_executor.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/feature_type.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/eval/tensor/default_tensor.h> #include <vespa/eval/tensor/tensor_mapper.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <memory> -#include "utils.h" -#include "valuefeature.h" #include <vespa/eval/eval/value_type.h> -#include <vespa/searchlib/fef/feature_type.h> +#include <memory> using namespace search::fef; using namespace search::fef::indexproperties; diff --git a/searchlib/src/vespa/searchlib/features/queryterm.cpp b/searchlib/src/vespa/searchlib/features/queryterm.cpp index f8c659be359..1ac25428a4c 100644 --- a/searchlib/src/vespa/searchlib/features/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/features/queryterm.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.queryterm"); #include "queryterm.h" #include "utils.h" diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp index 50cd4881817..2c5fa6e3032 100644 --- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp @@ -1,17 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.querytermcountfeature"); - +#include "querytermcountfeature.h" +#include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> -#include "querytermcountfeature.h" -#include "valuefeature.h" using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index b27bad7b98f..44bbe0db6e3 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <cmath> -LOG_SETUP(".features.randomnormalfeature"); -#include <vespa/searchlib/fef/properties.h> #include "random_normal_feature.h" #include "utils.h" +#include <vespa/searchlib/fef/properties.h> +#include <vespa/fastos/time.h> +#include <cmath> + +#include <vespa/log/log.h> +LOG_SETUP(".features.randomnormalfeature"); namespace search { namespace features { @@ -20,8 +21,7 @@ RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double st _spare(0.0) { - LOG(debug, "RandomNormalExecutor: seed=%" PRIu64 ", mean=%f, stddev=%f", - seed, mean, stddev); + LOG(debug, "RandomNormalExecutor: seed=%zu, mean=%f, stddev=%f", seed, mean, stddev); _rnd.srand48(seed); } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index 16fb4a99edb..19da7a19895 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.randomfeature"); #include "randomfeature.h" #include "utils.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fastos/time.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.randomfeature"); namespace search { namespace features { @@ -16,8 +18,7 @@ RandomExecutor::RandomExecutor(uint64_t seed, uint64_t matchSeed) : _matchRnd(), _matchSeed(matchSeed) { - LOG(debug, "RandomExecutor: seed=%" PRIu64 ", matchSeed=%" PRIu64, - seed, matchSeed); + LOG(debug, "RandomExecutor: seed=%" PRIu64 ", matchSeed=%" PRIu64, seed, matchSeed); _rnd.srand48(seed); } @@ -27,7 +28,6 @@ RandomExecutor::execute(uint32_t docId) feature_t rndScore = _rnd.lrand48() / (feature_t)0x80000000u; // 2^31 _matchRnd.srand48(_matchSeed + docId); feature_t matchRndScore = _matchRnd.lrand48() / (feature_t)0x80000000u; // 2^31 - //LOG(debug, "execute: %f", rndScore); outputs().set_number(0, rndScore); outputs().set_number(1, matchRndScore); } @@ -37,14 +37,12 @@ RandomBlueprint::RandomBlueprint() : search::fef::Blueprint("random"), _seed(0) { - // empty } void RandomBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, search::fef::IDumpFeatureVisitor &) const { - // empty } search::fef::Blueprint::UP diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/feature_name_extractor.cpp b/searchlib/src/vespa/searchlib/features/rankingexpression/feature_name_extractor.cpp index 4dc5124df67..3c453ee221c 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/feature_name_extractor.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/feature_name_extractor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "feature_name_extractor.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 07341ffd196..b14d44ee9b6 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/features/rankingexpression/feature_name_extractor.h> #include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/param_usage.h> #include <vespa/log/log.h> LOG_SETUP(".features.rankingexpression"); @@ -244,7 +245,8 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, // avoid costly compilation when only verifying setup if (env.getFeatureMotivation() != env.FeatureMotivation::VERIFY_SETUP) { if (do_compile) { - if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties())) { + bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function); + if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) { _compile_token = CompileCache::compile(rank_function, PassParams::LAZY); } else { _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY); diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index efee81ffd6c..0bed17a52e5 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.raw_score_feature"); #include "raw_score_feature.h" #include "utils.h" diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp index aba27c891ed..e185faff7ae 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.reverseproximity"); +#include "reverseproximityfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "reverseproximityfeature.h" -#include "utils.h" namespace search { namespace features { @@ -59,11 +56,9 @@ ReverseProximityExecutor::execute(uint32_t docId) } } } - //LOG(debug, "Initial guess; posA is '%u' and posB is '%u'.", posA, posB); // _P_A_R_A_N_O_I_A_ if (!itA.valid() || !itB.valid()) { - //LOG(debug, "Initial guess is invalid."); outputs().set_number(0, util::FEATURE_MAX); // out outputs().set_number(1, util::FEATURE_MIN); // posA outputs().set_number(2, util::FEATURE_MAX); // posB diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index e3dbda6a941..7ddf712d6c7 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "setup.h" #include "agefeature.h" diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp index 3fddc81268b..8f88b6a03da 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.subqueries_feature"); #include "subqueries_feature.h" #include "utils.h" diff --git a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp index b1eee1bc7ad..4eae5b8f950 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_attribute_executor.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_attribute_executor.h" #include <vespa/searchlib/tensor/tensor_attribute.h> diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp index bd8b4122df6..330275cb9cd 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp @@ -1,19 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.tensor_from_labels_feature"); - #include "tensor_from_labels_feature.h" #include "array_parser.hpp" #include "constant_tensor_executor.h" #include "tensor_from_attribute_executor.h" +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/feature_type.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/searchlib/fef/properties.h> #include <vespa/eval/tensor/default_tensor.h> #include <vespa/eval/eval/value_type.h> -#include <vespa/searchlib/fef/feature_type.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.tensor_from_labels_feature"); using namespace search::fef; using search::attribute::IAttributeVector; diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp index 7430fe69cab..5dfb8ce8cdc 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.tensor_from_weighted_set_feature"); - #include "tensor_from_weighted_set_feature.h" #include "constant_tensor_executor.h" @@ -11,15 +7,17 @@ LOG_SETUP(".features.tensor_from_weighted_set_feature"); #include "tensor_from_attribute_executor.h" #include "weighted_set_parser.hpp" +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/feature_type.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/searchlib/fef/properties.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/default_tensor.h> -#include <memory> #include <vespa/eval/eval/value_type.h> -#include <vespa/searchlib/fef/feature_type.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.tensor_from_weighted_set_feature"); using namespace search::fef; using search::attribute::IAttributeVector; diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index a51c16ae797..70c1276abf0 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "term_field_md_feature.h" +#include "utils.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> -#include "term_field_md_feature.h" -#include "utils.h" -LOG_SETUP(".features.term_field_md_feature"); using namespace search::fef; @@ -23,10 +20,10 @@ TermFieldMdExecutor::TermFieldMdExecutor(const search::fef::IQueryEnvironment &e { for (uint32_t i = 0; i < env.getNumTerms(); ++i) { const search::fef::ITermData *td = env.getTerm(i); - LOG_ASSERT(td != 0); + assert(td != 0); const search::fef::ITermFieldData *tfd = td->lookupField(fieldId); if (tfd != 0) { - LOG_ASSERT(tfd->getHandle() != search::fef::IllegalHandle); + assert(tfd->getHandle() != search::fef::IllegalHandle); _terms.push_back(std::make_pair(tfd->getHandle(), td->getWeight())); } } @@ -96,7 +93,7 @@ TermFieldMdBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { _field = params[0].asField(); - LOG_ASSERT(_field != 0); + assert(_field != 0); describeOutput("score", "The term field match score"); describeOutput("terms", "The number of ranked terms searching this field"); diff --git a/searchlib/src/vespa/searchlib/features/termdistancecalculator.cpp b/searchlib/src/vespa/searchlib/features/termdistancecalculator.cpp index ef8d1005a17..9689dd3efca 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancecalculator.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancecalculator.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.termdistancecalculator"); +#include "termdistancecalculator.h" #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/itermdata.h> -#include "termdistancecalculator.h" using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/termdistancecalculator.h b/searchlib/src/vespa/searchlib/features/termdistancecalculator.h index ae22d6fe38d..94675d591dc 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancecalculator.h +++ b/searchlib/src/vespa/searchlib/features/termdistancecalculator.h @@ -5,6 +5,12 @@ #include "queryterm.h" namespace search { + +namespace fef { + class TermFieldMatchData; + class MatchData; +} + namespace features { /** @@ -60,8 +66,8 @@ public: }; private: - static void findBest(const search::fef::TermFieldMatchData *tmdX, - const search::fef::TermFieldMatchData *tmdY, + static void findBest(const fef::TermFieldMatchData *tmdX, + const fef::TermFieldMatchData *tmdY, uint32_t numTermsX, uint32_t & bestDist, uint32_t & bestPos); @@ -73,9 +79,8 @@ public: * NB: Both query terms must have attached term fields with valid term field handles. **/ static void run(const QueryTerm &termX, const QueryTerm &termY, - const search::fef::MatchData & match, uint32_t docId, Result & r); + const fef::MatchData & match, uint32_t docId, Result & r); }; } // namespace features } // namespace search - diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index fd7bef0f201..8a512c48bd9 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -1,13 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.nativeproximityfeature"); -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/properties.h> #include "termdistancefeature.h" #include "valuefeature.h" #include "utils.h" +#include <vespa/searchlib/fef/fieldinfo.h> +#include <vespa/searchlib/fef/properties.h> using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp index da730ccc996..a414c217d7b 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp @@ -1,16 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.termeditdistance"); +#include "termeditdistancefeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "termeditdistancefeature.h" -#include "utils.h" + +#include <vespa/log/log.h> +LOG_SETUP(".features.termeditdistance"); namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp index a7ffb9a7bee..a47f8e47553 100644 --- a/searchlib/src/vespa/searchlib/features/termfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.termfeature"); +#include "termfeature.h" +#include "utils.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/vespalib/util/stringfmt.h> -#include "termfeature.h" -#include "utils.h" using namespace search::fef; diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp index 07b2904bf2a..91cdc888285 100644 --- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp @@ -1,18 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.terminfo"); - +#include "terminfofeature.h" +#include "valuefeature.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/handle.h> -#include <sstream> -#include "terminfofeature.h" -#include "valuefeature.h" namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp index 8340019cb02..c722cf1e74b 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".features.textsimilarity"); #include "text_similarity_feature.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/features/weighted_set_parser.cpp b/searchlib/src/vespa/searchlib/features/weighted_set_parser.cpp index 3d2f03d54eb..c426d2a85d5 100644 --- a/searchlib/src/vespa/searchlib/features/weighted_set_parser.cpp +++ b/searchlib/src/vespa/searchlib/features/weighted_set_parser.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "weighted_set_parser.h" + #include <vespa/log/log.h> LOG_SETUP(".features.weighted_set_parser"); -#include "weighted_set_parser.h" - namespace search { namespace features { diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.cpp b/searchlib/src/vespa/searchlib/fef/blueprint.cpp index c9a9b94eb44..ccea604b9b2 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprint.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.blueprint"); #include "blueprint.h" #include "parametervalidator.h" +#include <vespa/log/log.h> +LOG_SETUP(".fef.blueprint"); + namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp index af03194abb7..1bda84c9d16 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "blueprintfactory.h" + #include <vespa/log/log.h> LOG_SETUP(".fef.blueprintfactory"); -#include "blueprintfactory.h" namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/create-class-cpp.sh b/searchlib/src/vespa/searchlib/fef/create-class-cpp.sh index e47cc402dca..d9df58ae2b7 100755 --- a/searchlib/src/vespa/searchlib/fef/create-class-cpp.sh +++ b/searchlib/src/vespa/searchlib/fef/create-class-cpp.sh @@ -8,13 +8,9 @@ name=`echo $class | tr 'A-Z' 'a-z'` cat <<EOF // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP(".fef.$name"); -#include <vespa/fastos/fastos.h> #include "$name.h" -namespace search { -namespace fef { +namespace search::fef { $class::$class() { @@ -24,6 +20,5 @@ $class::~$class() { } -} // namespace fef -} // namespace search +} EOF diff --git a/searchlib/src/vespa/searchlib/fef/feature_type.cpp b/searchlib/src/vespa/searchlib/fef/feature_type.cpp index 63434ec6bb7..b4fdbb927a1 100644 --- a/searchlib/src/vespa/searchlib/fef/feature_type.cpp +++ b/searchlib/src/vespa/searchlib/fef/feature_type.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "feature_type.h" +#include <cassert> namespace search { namespace fef { @@ -22,5 +22,11 @@ FeatureType::object(const TYPE &type_in) return FeatureType(std::make_unique<TYPE>(type_in)); } +const FeatureType::TYPE & +FeatureType::type() const { + assert(_type); + return *_type; +} + } // namespace fef } // namespace search diff --git a/searchlib/src/vespa/searchlib/fef/feature_type.h b/searchlib/src/vespa/searchlib/fef/feature_type.h index 74bdc4149eb..775d89cc2e6 100644 --- a/searchlib/src/vespa/searchlib/fef/feature_type.h +++ b/searchlib/src/vespa/searchlib/fef/feature_type.h @@ -28,10 +28,7 @@ public: FeatureType(const FeatureType &rhs); bool is_object() const { return (_type.get() != nullptr); } operator bool() const { return is_object(); } - const TYPE &type() const { - assert(_type); - return *_type; - } + const TYPE &type() const; static const FeatureType &number() { return _number; } static FeatureType object(const TYPE &type_in); }; diff --git a/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp b/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp index 292f5ac5bcc..1416f8673c7 100644 --- a/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/featurenamebuilder.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "featurenamebuilder.h" #include "featurenameparser.h" diff --git a/searchlib/src/vespa/searchlib/fef/featurenameparser.cpp b/searchlib/src/vespa/searchlib/fef/featurenameparser.cpp index fca21041f87..698355b25a7 100644 --- a/searchlib/src/vespa/searchlib/fef/featurenameparser.cpp +++ b/searchlib/src/vespa/searchlib/fef/featurenameparser.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.featurenameparser"); -#include <vespa/vespalib/util/stringfmt.h> #include "featurenameparser.h" #include "featurenamebuilder.h" +#include <vespa/vespalib/util/stringfmt.h> + +#include <vespa/log/log.h> +LOG_SETUP(".fef.featurenameparser"); namespace { diff --git a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp index 364abea4942..f2a0c2b9501 100644 --- a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp +++ b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "featureoverrider.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/fef.cpp b/searchlib/src/vespa/searchlib/fef/fef.cpp index 6a29c098479..1a4446c2a82 100644 --- a/searchlib/src/vespa/searchlib/fef/fef.cpp +++ b/searchlib/src/vespa/searchlib/fef/fef.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fef.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/fieldinfo.cpp b/searchlib/src/vespa/searchlib/fef/fieldinfo.cpp index 234352ddb31..13d746ba5fa 100644 --- a/searchlib/src/vespa/searchlib/fef/fieldinfo.cpp +++ b/searchlib/src/vespa/searchlib/fef/fieldinfo.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fieldinfo.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.cpp b/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.cpp index 8994c9492a7..c938a427006 100644 --- a/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.cpp +++ b/searchlib/src/vespa/searchlib/fef/fieldpositionsiterator.cpp @@ -1,14 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fieldpositionsiterator.h" #include <vespa/searchlib/common/fslimits.h> namespace search { namespace fef { -const uint32_t FieldPositionsIterator::UNKNOWN_LENGTH = -SEARCHLIB_FEF_UNKNOWN_FIELD_LENGTH; +const uint32_t FieldPositionsIterator::UNKNOWN_LENGTH = SEARCHLIB_FEF_UNKNOWN_FIELD_LENGTH; } // namespace fef } // namespace search diff --git a/searchlib/src/vespa/searchlib/fef/filetablefactory.cpp b/searchlib/src/vespa/searchlib/fef/filetablefactory.cpp index c113efe33d9..6d61e1f21c6 100644 --- a/searchlib/src/vespa/searchlib/fef/filetablefactory.cpp +++ b/searchlib/src/vespa/searchlib/fef/filetablefactory.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.filetablefactory"); #include "filetablefactory.h" #include <fstream> +#include <vespa/log/log.h> +LOG_SETUP(".fef.filetablefactory"); + namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/functiontablefactory.cpp b/searchlib/src/vespa/searchlib/fef/functiontablefactory.cpp index 55467b81cb6..ab5f1dc6b74 100644 --- a/searchlib/src/vespa/searchlib/fef/functiontablefactory.cpp +++ b/searchlib/src/vespa/searchlib/fef/functiontablefactory.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.functiontablefactory"); +#include "functiontablefactory.h" #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <cmath> -#include "functiontablefactory.h" +#include <vespa/log/log.h> +LOG_SETUP(".fef.functiontablefactory"); namespace { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index b75a0f33393..d887d279e95 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -77,12 +77,11 @@ checkIfTrue(const Properties &props, const vespalib::string &name, namespace eval { const vespalib::string LazyExpressions::NAME("vespa.eval.lazy_expressions"); -const vespalib::string LazyExpressions::DEFAULT_VALUE("false"); bool -LazyExpressions::check(const Properties &props) +LazyExpressions::check(const Properties &props, bool default_value) { - return checkIfTrue(props, NAME, DEFAULT_VALUE); + return lookupBool(props, NAME, default_value); } } // namespace eval diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index 4a97535879b..02942b1520a 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -26,8 +26,7 @@ namespace eval { // lazy evaluation of expressions. affects rank/summary/dump struct LazyExpressions { static const vespalib::string NAME; - static const vespalib::string DEFAULT_VALUE; - static bool check(const Properties &props); + static bool check(const Properties &props, bool default_value); }; } // namespace eval diff --git a/searchlib/src/vespa/searchlib/fef/itermdata.h b/searchlib/src/vespa/searchlib/fef/itermdata.h index dbcb3867923..efdf394ea6e 100644 --- a/searchlib/src/vespa/searchlib/fef/itermdata.h +++ b/searchlib/src/vespa/searchlib/fef/itermdata.h @@ -2,9 +2,9 @@ #pragma once -#include "handle.h" #include "itermfielddata.h" #include <vespa/searchlib/query/weight.h> +#include <cstddef> namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/location.cpp b/searchlib/src/vespa/searchlib/fef/location.cpp index 9bed7305a1e..34f77f02b82 100644 --- a/searchlib/src/vespa/searchlib/fef/location.cpp +++ b/searchlib/src/vespa/searchlib/fef/location.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "location.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/parameter.cpp b/searchlib/src/vespa/searchlib/fef/parameter.cpp index 583061ad383..1952967c439 100644 --- a/searchlib/src/vespa/searchlib/fef/parameter.cpp +++ b/searchlib/src/vespa/searchlib/fef/parameter.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "parameter.h" namespace search { @@ -11,7 +10,7 @@ Parameter::Parameter(ParameterType::Enum type, const vespalib::string & value) : _stringVal(value), _doubleVal(0), _intVal(0), - _fieldVal(NULL) + _fieldVal(nullptr) { } diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.cpp b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.cpp index 5262a41f734..3a6e4147e81 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.cpp +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "parameterdescriptions.h" +#include <cassert> namespace search { namespace fef { @@ -51,5 +51,20 @@ ParameterDescriptions::desc(size_t tag) { return *this; } +void +ParameterDescriptions::addParameter(const ParamDescItem ¶m) { + assert(!_descriptions.empty()); + assert(!getCurrent().hasRepeat()); + getCurrent().addParameter(param); +} + +ParameterDescriptions & +ParameterDescriptions::repeat(size_t n) { + assert(!_descriptions.empty()); + assert(getCurrent().getParams().size() >= n); + getCurrent().setRepeat(n); + return *this; +} + } // namespace fef } // namespace search diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h index d91e96b0e59..dcd2a44d75f 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h @@ -3,6 +3,7 @@ #pragma once #include <vector> +#include <cstddef> namespace search { namespace fef { @@ -99,11 +100,7 @@ private: size_t _nextTag; Description & getCurrent() { return _descriptions.back(); } - void addParameter(const ParamDescItem ¶m) { - assert(!_descriptions.empty()); - assert(!getCurrent().hasRepeat()); - getCurrent().addParameter(param); - } + void addParameter(const ParamDescItem ¶m); void addParameter(ParameterType::Enum type, ParameterCollection::Enum collection) { addParameter(ParamDescItem(type, collection)); } @@ -175,16 +172,8 @@ public: /** * Sets the repeat number on the current description. */ - ParameterDescriptions & repeat(size_t n = 1) { - assert(!_descriptions.empty()); - assert(getCurrent().getParams().size() >= n); - getCurrent().setRepeat(n); - return *this; - } + ParameterDescriptions & repeat(size_t n = 1); }; - - } // namespace fef } // namespace search - diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp index 696e8a18dfc..3fb8305fcea 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <boost/lexical_cast.hpp> -#include <vespa/vespalib/util/vstringfmt.h> -#include "fieldinfo.h" -#include "fieldtype.h" #include "parametervalidator.h" +#include "fieldtype.h" +#include "fieldinfo.h" +#include <vespa/vespalib/util/vstringfmt.h> +#include <boost/lexical_cast.hpp> using vespalib::make_vespa_string; diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp index 23f79080cd3..1a8201c2bc6 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.phrasesplitter"); #include "phrasesplitter.h" namespace search { @@ -51,7 +48,7 @@ PhraseSplitter::PhraseSplitter(const IQueryEnvironment & queryEnv, for (uint32_t i = 0; i < queryEnv.getNumTerms(); ++i) { const ITermData *td = queryEnv.getTerm(i); - LOG_ASSERT(td != NULL); + assert(td != NULL); considerTerm(i, *td, phraseTerms, fieldId); numHandles += td->numFields(); } @@ -100,7 +97,7 @@ PhraseSplitter::update() for (uint32_t i = 0; i < _copyInfo.size(); ++i) { const TermFieldMatchData *src = _matchData->resolveTermField(_copyInfo[i].orig_handle); TermFieldMatchData *dst = resolveSplittedTermField(_copyInfo[i].split_handle); - LOG_ASSERT(src != NULL && dst != NULL); + assert(src != NULL && dst != NULL); copyTermFieldMatchData(*dst, *src, _copyInfo[i].offsetInPhrase); } diff --git a/searchlib/src/vespa/searchlib/fef/queryproperties.cpp b/searchlib/src/vespa/searchlib/fef/queryproperties.cpp index a5dd5dc9229..41e8fc7053d 100644 --- a/searchlib/src/vespa/searchlib/fef/queryproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/queryproperties.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "queryproperties.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp b/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp index 97ec1f8cca3..34a9bd63780 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.simpletermdata"); #include "simpletermdata.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/simpletermfielddata.cpp b/searchlib/src/vespa/searchlib/fef/simpletermfielddata.cpp index 582e5e330d6..6318fd547eb 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermfielddata.cpp +++ b/searchlib/src/vespa/searchlib/fef/simpletermfielddata.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.simpletermfielddata"); #include "simpletermfielddata.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/symmetrictable.cpp b/searchlib/src/vespa/searchlib/fef/symmetrictable.cpp index 29818a9f416..5f5bcf71263 100644 --- a/searchlib/src/vespa/searchlib/fef/symmetrictable.cpp +++ b/searchlib/src/vespa/searchlib/fef/symmetrictable.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "symmetrictable.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/table.cpp b/searchlib/src/vespa/searchlib/fef/table.cpp index c32cd233937..f1eb23ccb7a 100644 --- a/searchlib/src/vespa/searchlib/fef/table.cpp +++ b/searchlib/src/vespa/searchlib/fef/table.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "table.h" #include <limits> diff --git a/searchlib/src/vespa/searchlib/fef/table.h b/searchlib/src/vespa/searchlib/fef/table.h index a2203b83041..03a0310d0bf 100644 --- a/searchlib/src/vespa/searchlib/fef/table.h +++ b/searchlib/src/vespa/searchlib/fef/table.h @@ -3,9 +3,9 @@ #pragma once #include <vector> +#include <memory> -namespace search { -namespace fef { +namespace search::fef { /** * This class represents a rank table with double values. @@ -60,6 +60,4 @@ public: } }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index e4c7d92dce4..b6bec259450 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.termfieldmatchdata"); #include "termfieldmatchdata.h" #include "fieldinfo.h" #include <algorithm> diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.cpp index f76c785b616..fcfa73d9456 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataposition.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.termfieldmatchdataposition"); #include "termfieldmatchdataposition.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp index 7515a6338e0..f8155495e2a 100644 --- a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "dummy_dependency_handler.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp index 1a359b4056e..08a0017c3e2 100644 --- a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp @@ -3,6 +3,7 @@ #include "ftlib.h" #include "dummy_dependency_handler.h" #include <vespa/searchlib/features/utils.h> +#include <vespa/vespalib/util/stringfmt.h> #include <boost/tokenizer.hpp> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp index dbe3ea52078..c173cb474e7 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "indexenvironmentbuilder.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp index 1f59cfb52b8..02782565f80 100644 --- a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".fef.matchdatabuilder"); diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp index ce0878c7306..96075188fc6 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp @@ -1,12 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.cfgvalue"); - +#include "cfgvalue.h" #include <vespa/searchlib/fef/properties.h> #include <sstream> -#include "cfgvalue.h" namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp index f79209270b8..7e6aabeebf1 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.chain"); #include "chain.h" - #include <sstream> namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp index 5fe7539b834..82d4e4e8865 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp @@ -1,12 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.double"); #include "double.h" - -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchlib/fef/featurenamebuilder.h> +#include <vespa/vespalib/stllike/asciistream.h> namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp index 60fafd8ae90..6a2fc8c64ac 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp @@ -1,13 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.query"); - +#include "query.h" #include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/fef/properties.h> #include <sstream> -#include "query.h" namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/setup.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/setup.cpp index 94a74947cea..8ca6692360d 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/setup.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/setup.cpp @@ -1,12 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.setup"); - -#include <vespa/searchlib/fef/blueprint.h> -#include <string> - #include "cfgvalue.h" #include "chain.h" #include "double.h" @@ -14,6 +7,7 @@ LOG_SETUP(".fef.setup"); #include "setup.h" #include "staticrank.h" #include "sum.h" +#include <vespa/searchlib/fef/blueprint.h> namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp index 238f0ca0dd4..003fc3c0a5a 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.staticrank"); -#include <vespa/searchcommon/attribute/attributecontent.h> #include "staticrank.h" +#include <vespa/searchcommon/attribute/attributecontent.h> namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp index 2e20ea12efe..775d2203915 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.sum"); #include "sum.h" #include <vespa/searchlib/fef/featurenamebuilder.h> diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp index 7db5110e627..107ad9914b2 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "queryenvironmentbuilder.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/fef/test/rankresult.cpp b/searchlib/src/vespa/searchlib/fef/test/rankresult.cpp index 218153db489..167f569995f 100644 --- a/searchlib/src/vespa/searchlib/fef/test/rankresult.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/rankresult.cpp @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.rankresult"); #include "rankresult.h" #include <cmath> -#include <iostream> +#include <ostream> + +#include <vespa/log/log.h> +LOG_SETUP(".fef.rankresult"); namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/fef/utils.cpp b/searchlib/src/vespa/searchlib/fef/utils.cpp index 70396857759..06eed77a3a9 100644 --- a/searchlib/src/vespa/searchlib/fef/utils.cpp +++ b/searchlib/src/vespa/searchlib/fef/utils.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "utils.h" #include <vector> diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp index ebfdf1622ba..e2fa8faa5b8 100644 --- a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp +++ b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.verify_feature"); #include "verify_feature.h" #include "blueprintresolver.h" +#include <vespa/log/log.h> +LOG_SETUP(".fef.verify_feature"); + namespace search { namespace fef { diff --git a/searchlib/src/vespa/searchlib/grouping/groupandcollectengine.cpp b/searchlib/src/vespa/searchlib/grouping/groupandcollectengine.cpp index 6f06960d5c5..554737cc481 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupandcollectengine.cpp +++ b/searchlib/src/vespa/searchlib/grouping/groupandcollectengine.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/grouping/groupandcollectengine.h> +#include "groupandcollectengine.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/grouping/groupingengine.cpp b/searchlib/src/vespa/searchlib/grouping/groupingengine.cpp index a8f12ce9fd4..010d947590e 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupingengine.cpp +++ b/searchlib/src/vespa/searchlib/grouping/groupingengine.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/grouping/groupingengine.h> -#include <vespa/searchlib/grouping/groupandcollectengine.h> + +#include "groupingengine.h" +#include "groupandcollectengine.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/index/bitvectorkeys.h b/searchlib/src/vespa/searchlib/index/bitvectorkeys.h index 2a1e33026af..05e8ec832ce 100644 --- a/searchlib/src/vespa/searchlib/index/bitvectorkeys.h +++ b/searchlib/src/vespa/searchlib/index/bitvectorkeys.h @@ -17,7 +17,7 @@ public: uint32_t _numDocs; uint32_t _pad; - BitVectorWordSingleKey(void) + BitVectorWordSingleKey() : _wordNum(0), _numDocs(0), _pad(0) diff --git a/searchlib/src/vespa/searchlib/index/docbuilder.cpp b/searchlib/src/vespa/searchlib/index/docbuilder.cpp index 167aee6b194..2e0e7364d68 100644 --- a/searchlib/src/vespa/searchlib/index/docbuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/docbuilder.cpp @@ -270,7 +270,7 @@ DocBuilder::IndexFieldHandle::addStr(const vespalib::string &val) void -DocBuilder::IndexFieldHandle::addSpace(void) +DocBuilder::IndexFieldHandle::addSpace() { addNoWordStr(" "); } @@ -345,7 +345,7 @@ DocBuilder::IndexFieldHandle::addSpan(size_t start, size_t len) void -DocBuilder::IndexFieldHandle::addSpan(void) +DocBuilder::IndexFieldHandle::addSpan() { size_t endPos = _strSymbols; assert(endPos > _spanStart); @@ -355,7 +355,7 @@ DocBuilder::IndexFieldHandle::addSpan(void) void -DocBuilder::IndexFieldHandle::addSpaceTokenAnnotation(void) +DocBuilder::IndexFieldHandle::addSpaceTokenAnnotation() { assert(_spanTree.get() != NULL); assert(_lastSpan != NULL); @@ -364,7 +364,7 @@ DocBuilder::IndexFieldHandle::addSpaceTokenAnnotation(void) void -DocBuilder::IndexFieldHandle::addNumericTokenAnnotation(void) +DocBuilder::IndexFieldHandle::addNumericTokenAnnotation() { assert(_spanTree.get() != NULL); assert(_lastSpan != NULL); @@ -373,7 +373,7 @@ DocBuilder::IndexFieldHandle::addNumericTokenAnnotation(void) void -DocBuilder::IndexFieldHandle::addAlphabeticTokenAnnotation(void) +DocBuilder::IndexFieldHandle::addAlphabeticTokenAnnotation() { assert(_spanTree.get() != NULL); assert(_lastSpan != NULL); @@ -382,7 +382,7 @@ DocBuilder::IndexFieldHandle::addAlphabeticTokenAnnotation(void) void -DocBuilder::IndexFieldHandle::addTermAnnotation(void) +DocBuilder::IndexFieldHandle::addTermAnnotation() { assert(_spanTree.get() != NULL); assert(_lastSpan != NULL); @@ -402,7 +402,7 @@ DocBuilder::IndexFieldHandle::addTermAnnotation(const vespalib::string &val) void -DocBuilder::IndexFieldHandle::onEndElement(void) +DocBuilder::IndexFieldHandle::onEndElement() { // Flush data for index field. assert(_subField.empty()); @@ -434,7 +434,7 @@ DocBuilder::IndexFieldHandle::onEndElement(void) void -DocBuilder::IndexFieldHandle::onEndField(void) +DocBuilder::IndexFieldHandle::onEndField() { if (_sfield.getCollectionType() == CollectionType::SINGLE) onEndElement(); @@ -442,7 +442,7 @@ DocBuilder::IndexFieldHandle::onEndField(void) void -DocBuilder::IndexFieldHandle::startAnnotate(void) +DocBuilder::IndexFieldHandle::startAnnotate() { SpanList::UP span_list(new SpanList); _spanList = span_list.get(); @@ -475,7 +475,7 @@ DocBuilder::IndexFieldHandle::startSubField(const vespalib::string &subField) void -DocBuilder::IndexFieldHandle::endSubField(void) +DocBuilder::IndexFieldHandle::endSubField() { assert(!_subField.empty()); assert(_uriField); @@ -739,7 +739,7 @@ DocBuilder::addStr(const vespalib::string & str) } DocBuilder & -DocBuilder::addSpace(void) +DocBuilder::addSpace() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addSpace(); @@ -815,7 +815,7 @@ DocBuilder::addSpan(size_t start, size_t len) DocBuilder & -DocBuilder::addSpan(void) +DocBuilder::addSpan() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addSpan(); @@ -824,7 +824,7 @@ DocBuilder::addSpan(void) DocBuilder & -DocBuilder::addSpaceTokenAnnotation(void) +DocBuilder::addSpaceTokenAnnotation() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addSpaceTokenAnnotation(); @@ -833,7 +833,7 @@ DocBuilder::addSpaceTokenAnnotation(void) DocBuilder & -DocBuilder::addNumericTokenAnnotation(void) +DocBuilder::addNumericTokenAnnotation() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addNumericTokenAnnotation(); @@ -842,7 +842,7 @@ DocBuilder::addNumericTokenAnnotation(void) DocBuilder & -DocBuilder::addAlphabeticTokenAnnotation(void) +DocBuilder::addAlphabeticTokenAnnotation() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addAlphabeticTokenAnnotation(); @@ -851,7 +851,7 @@ DocBuilder::addAlphabeticTokenAnnotation(void) DocBuilder& -DocBuilder::addTermAnnotation(void) +DocBuilder::addTermAnnotation() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->addTermAnnotation(); @@ -896,7 +896,7 @@ DocBuilder::startSubField(const vespalib::string &subField) DocBuilder & -DocBuilder::endSubField(void) +DocBuilder::endSubField() { assert(_currDoc != NULL); _currDoc->getFieldHandle()->endSubField(); diff --git a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp index c6530209b0b..36fbc0a4ce8 100644 --- a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp @@ -67,7 +67,7 @@ insertStructType(document::DocumenttypesConfig::Documenttype & cfg, } -DocTypeBuilder::UriField::UriField(void) +DocTypeBuilder::UriField::UriField() : _all(Schema::UNKNOWN_FIELD_ID), _scheme(Schema::UNKNOWN_FIELD_ID), _host(Schema::UNKNOWN_FIELD_ID), diff --git a/searchlib/src/vespa/searchlib/index/doctypebuilder.h b/searchlib/src/vespa/searchlib/index/doctypebuilder.h index d5f2b497b3a..ae7f44441c3 100644 --- a/searchlib/src/vespa/searchlib/index/doctypebuilder.h +++ b/searchlib/src/vespa/searchlib/index/doctypebuilder.h @@ -37,7 +37,7 @@ public: const Schema::CollectionType &collectionType); public: - UriField(void); + UriField(); bool broken(const Schema &schema, const Schema::CollectionType &collectionType) const; bool valid(const Schema &schema, const Schema::CollectionType &collectionType) const; diff --git a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp index 53673cac9cc..6d7dace1f8c 100644 --- a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp @@ -1,22 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.dummyfileheadercontext"); #include "dummyfileheadercontext.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/util/host_name.h> +#include <unistd.h> -namespace search -{ - -namespace index -{ +namespace search::index { vespalib::string DummyFileHeaderContext::_creator; -DummyFileHeaderContext::DummyFileHeaderContext(void) +DummyFileHeaderContext::DummyFileHeaderContext() : common::FileHeaderContext(), _disableFileName(false), _hostName(), @@ -27,13 +21,13 @@ DummyFileHeaderContext::DummyFileHeaderContext(void) } -DummyFileHeaderContext::~DummyFileHeaderContext(void) +DummyFileHeaderContext::~DummyFileHeaderContext() { } void -DummyFileHeaderContext::disableFileName(void) +DummyFileHeaderContext::disableFileName() { _disableFileName = true; } @@ -65,7 +59,4 @@ DummyFileHeaderContext::setCreator(const vespalib::string &creator) _creator = creator; } - -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/indexbuilder.cpp b/searchlib/src/vespa/searchlib/index/indexbuilder.cpp index aaefb0e4ff8..6ab25c1cce1 100644 --- a/searchlib/src/vespa/searchlib/index/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/indexbuilder.cpp @@ -1,28 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.indexbuilder"); #include "indexbuilder.h" -namespace search -{ - -namespace index -{ - +namespace search::index { IndexBuilder::IndexBuilder(const Schema &schema) : _schema(schema) -{ -} +{ } +IndexBuilder::~IndexBuilder() { } -IndexBuilder::~IndexBuilder(void) -{ } - - -} // namespace index - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/index/indexbuilder.h b/searchlib/src/vespa/searchlib/index/indexbuilder.h index f4688167b81..edbfd179b15 100644 --- a/searchlib/src/vespa/searchlib/index/indexbuilder.h +++ b/searchlib/src/vespa/searchlib/index/indexbuilder.h @@ -21,31 +21,31 @@ public: IndexBuilder(const Schema &schema); virtual - ~IndexBuilder(void); + ~IndexBuilder(); virtual void startWord(const vespalib::stringref & word) = 0; virtual void - endWord(void) = 0; + endWord() = 0; virtual void startDocument(uint32_t docId) = 0; virtual void - endDocument(void) = 0; + endDocument() = 0; virtual void startField(uint32_t fieldId) = 0; virtual void - endField(void) = 0; + endField() = 0; virtual void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) = 0; virtual void - endElement(void) = 0; + endElement() = 0; virtual void addOcc(const WordDocElementWordPosFeatures &features) = 0; diff --git a/searchlib/src/vespa/searchlib/index/olddictionaryfile.cpp b/searchlib/src/vespa/searchlib/index/olddictionaryfile.cpp index 7e194e3ead4..774b373115f 100644 --- a/searchlib/src/vespa/searchlib/index/olddictionaryfile.cpp +++ b/searchlib/src/vespa/searchlib/index/olddictionaryfile.cpp @@ -1,18 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.dictionaryfile"); #include "olddictionaryfile.h" -namespace search -{ - -namespace index -{ - +namespace search::index { -OldDictionaryIndexMapping::OldDictionaryIndexMapping(void) +OldDictionaryIndexMapping::OldDictionaryIndexMapping() : _fieldIdToLocalId(), _indexNames(), _indexIds(), @@ -21,7 +13,7 @@ OldDictionaryIndexMapping::OldDictionaryIndexMapping(void) } -OldDictionaryIndexMapping::~OldDictionaryIndexMapping(void) +OldDictionaryIndexMapping::~OldDictionaryIndexMapping() { } @@ -100,16 +92,13 @@ OldDictionaryIndexMapping::setupHelper(const Schema &schema) } -OldDictionaryFileSeqRead::~OldDictionaryFileSeqRead(void) +OldDictionaryFileSeqRead::~OldDictionaryFileSeqRead() { } -OldDictionaryFileSeqWrite::~OldDictionaryFileSeqWrite(void) +OldDictionaryFileSeqWrite::~OldDictionaryFileSeqWrite() { } - -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/olddictionaryfile.h b/searchlib/src/vespa/searchlib/index/olddictionaryfile.h index cd7d2171b61..3b5b587aff9 100644 --- a/searchlib/src/vespa/searchlib/index/olddictionaryfile.h +++ b/searchlib/src/vespa/searchlib/index/olddictionaryfile.h @@ -35,12 +35,12 @@ private: setupHelper(const Schema &schema); public: - OldDictionaryIndexMapping(void); + OldDictionaryIndexMapping(); - ~OldDictionaryIndexMapping(void); + ~OldDictionaryIndexMapping(); static uint32_t - noLocalId(void) + noLocalId() { return std::numeric_limits<uint32_t>::max(); } @@ -69,25 +69,25 @@ public: const std::vector<uint32_t> &indexes); const std::vector<uint32_t> & - getIndexIds(void) const + getIndexIds() const { return _indexIds; } const std::vector<uint32_t> & - getWashedIndexIds(void) const + getWashedIndexIds() const { return _washedIndexIds; } const std::vector<vespalib::string> & - getIndexNames(void) const + getIndexNames() const { return _indexNames; } uint32_t - getNumIndexes(void) const + getNumIndexes() const { return _indexIds.size(); } @@ -102,12 +102,12 @@ public: class OldDictionaryFileSeqRead { public: - OldDictionaryFileSeqRead(void) + OldDictionaryFileSeqRead() { } virtual - ~OldDictionaryFileSeqRead(void); + ~OldDictionaryFileSeqRead(); /** * Read word and counts. Only nonzero counts are returned. If at @@ -133,7 +133,7 @@ public: * Close dictionary file. */ virtual bool - close(void) = 0; + close() = 0; /* * Get visible indexes available in dictionary. @@ -142,13 +142,13 @@ public: getIndexes(std::vector<uint32_t> &indexes) = 0; static uint64_t - noWordNum(void) + noWordNum() { return 0u; } static uint64_t - noWordNumHigh(void) + noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } @@ -166,12 +166,12 @@ class OldDictionaryFileSeqWrite { protected: public: - OldDictionaryFileSeqWrite(void) + OldDictionaryFileSeqWrite() { } virtual - ~OldDictionaryFileSeqWrite(void); + ~OldDictionaryFileSeqWrite(); /** * Write word and counts. Only nonzero counts should be supplied. @@ -198,7 +198,7 @@ public: * Close dictionary file. */ virtual bool - close(void) = 0; + close() = 0; }; diff --git a/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp b/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp index 78c7d28905b..314476628a1 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistcountfile.cpp @@ -1,22 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.postinglistcountfile"); #include "postinglistcountfile.h" -namespace search -{ - -namespace index -{ +namespace search::index { -PostingListCountFileSeqRead::PostingListCountFileSeqRead(void) +PostingListCountFileSeqRead::PostingListCountFileSeqRead() { } -PostingListCountFileSeqRead::~PostingListCountFileSeqRead(void) +PostingListCountFileSeqRead::~PostingListCountFileSeqRead() { } @@ -29,12 +22,12 @@ getParams(PostingListParams ¶ms) } -PostingListCountFileSeqWrite::PostingListCountFileSeqWrite(void) +PostingListCountFileSeqWrite::PostingListCountFileSeqWrite() { } -PostingListCountFileSeqWrite::~PostingListCountFileSeqWrite(void) +PostingListCountFileSeqWrite::~PostingListCountFileSeqWrite() { } @@ -54,7 +47,4 @@ getParams(PostingListParams ¶ms) params.clear(); } - -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp b/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp index 8268b1e5c64..2679e33f2c2 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistcounts.cpp @@ -1,16 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.postinglistcounts"); #include <vespa/vespalib/objects/nbostream.h> #include "postinglistcounts.h" -namespace search -{ - -namespace index -{ +namespace search::index { void swap(PostingListCounts & a, PostingListCounts & b) { @@ -84,7 +77,4 @@ operator>>(nbostream &in, PostingListOffsetAndCounts &offsetAndCounts) return in; } - -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp index 677ca101d88..dc04b3fe716 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp @@ -1,24 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.postinglistfile"); #include "postinglistfile.h" +#include <vespa/fastos/file.h> -namespace search -{ - -namespace index -{ +namespace search::index { -PostingListFileSeqRead::PostingListFileSeqRead(void) +PostingListFileSeqRead::PostingListFileSeqRead() : _counts(), _residueDocs(0) { } -PostingListFileSeqRead::~PostingListFileSeqRead(void) +PostingListFileSeqRead::~PostingListFileSeqRead() { } @@ -47,13 +41,13 @@ getFeatureParams(PostingListParams ¶ms) } -PostingListFileSeqWrite::PostingListFileSeqWrite(void) +PostingListFileSeqWrite::PostingListFileSeqWrite() : _counts() { } -PostingListFileSeqWrite::~PostingListFileSeqWrite(void) +PostingListFileSeqWrite::~PostingListFileSeqWrite() { } @@ -91,13 +85,13 @@ getFeatureParams(PostingListParams ¶ms) PostingListFileRandRead:: -PostingListFileRandRead(void) +PostingListFileRandRead() : _memoryMapped(false) { } -PostingListFileRandRead::~PostingListFileRandRead(void) +PostingListFileRandRead::~PostingListFileRandRead() { } @@ -118,7 +112,7 @@ PostingListFileRandReadPassThrough(PostingListFileRandRead *lower, } -PostingListFileRandReadPassThrough::~PostingListFileRandReadPassThrough(void) +PostingListFileRandReadPassThrough::~PostingListFileRandReadPassThrough() { if (_ownLower) delete _lower; @@ -159,12 +153,9 @@ PostingListFileRandReadPassThrough::open(const vespalib::string &name, bool -PostingListFileRandReadPassThrough::close(void) +PostingListFileRandReadPassThrough::close() { return _lower->close(); } - -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 1fe82277de7..44b6b1ac088 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -180,7 +180,7 @@ public: */ virtual void getFeatureParams(PostingListParams ¶ms); - PostingListCounts &getCounts(void) { return _counts; } + PostingListCounts &getCounts() { return _counts; } }; @@ -269,4 +269,4 @@ public: } // namespace index -} // namespace search
\ No newline at end of file +} // namespace search diff --git a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp index 9c64587f7ee..ed0ed3eeee2 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp @@ -1,23 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.postinglistparams"); #include "postinglistparams.h" #include <sstream> -namespace -{ +namespace { vespalib::string empty; } -namespace search -{ - -namespace index -{ +namespace search::index { bool PostingListParams::isSet(const vespalib::string &key) const @@ -52,7 +44,7 @@ PostingListParams::getStr(const vespalib::string &key) const void -PostingListParams::clear(void) +PostingListParams::clear() { _map.clear(); } @@ -133,6 +125,4 @@ template void PostingListParams::get<uint64_t>(const vespalib::string &key, uint64_t &val) const; -} // namespace index - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/index/postinglistparams.h b/searchlib/src/vespa/searchlib/index/postinglistparams.h index d8424b81835..ad0c5d5f9af 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistparams.h +++ b/searchlib/src/vespa/searchlib/index/postinglistparams.h @@ -33,7 +33,7 @@ public: getStr(const vespalib::string &key) const; void - clear(void); + clear(); void erase(const vespalib::string &key); diff --git a/searchlib/src/vespa/searchlib/index/schemautil.cpp b/searchlib/src/vespa/searchlib/index/schemautil.cpp index 52c5a1caee4..544bb5f8a3c 100644 --- a/searchlib/src/vespa/searchlib/index/schemautil.cpp +++ b/searchlib/src/vespa/searchlib/index/schemautil.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <fstream> #include "schemautil.h" #include <set> -LOG_SETUP(".index.schemautil"); +#include <fstream> -namespace search { +#include <vespa/log/log.h> +LOG_SETUP(".index.schemautil"); -namespace index { +namespace search::index { using schema::DataType; @@ -207,6 +205,4 @@ SchemaUtil::getIndexIds(const Schema &schema, return true; } - -} // namespace search::index -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp b/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp index 665d377af99..063f177fd18 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/exceptions.h> #include "dictionary.h" - +#include "fieldinverter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/btree/btreenode.hpp> @@ -15,9 +11,9 @@ #include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btree.hpp> -#include "fieldinverter.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/exceptions.h> -LOG_SETUP(".memoryindex.dictionary"); namespace search { @@ -37,7 +33,7 @@ Dictionary::Dictionary(const Schema & schema) } } -Dictionary::~Dictionary(void) +Dictionary::~Dictionary() { } diff --git a/searchlib/src/vespa/searchlib/memoryindex/dictionary.h b/searchlib/src/vespa/searchlib/memoryindex/dictionary.h index b4093a05a43..88dad7b1b39 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/dictionary.h +++ b/searchlib/src/vespa/searchlib/memoryindex/dictionary.h @@ -22,7 +22,7 @@ private: public: Dictionary(const index::Schema &schema); - ~Dictionary(void); + ~Dictionary(); PostingList::Iterator find(const vespalib::stringref word, uint32_t fieldId) const { diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp index 3c9c8bb6ac4..bf20415843d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp @@ -1,13 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "document_remover.h" #include "i_document_remove_listener.h" #include "wordstore.h" #include <vespa/searchlib/common/sort.h> -LOG_SETUP(".memoryindex.document_remover"); - namespace search { namespace memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp index 7e6d2a998d5..0cb5a6377c3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp @@ -1,7 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "documentinverter.h" +#include "fieldinverter.h" +#include "urlfieldinverter.h" +#include "dictionary.h" +#include "ordereddocumentinserter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/searchlib/util/url.h> @@ -10,19 +13,12 @@ #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> #include <vespa/document/repo/fixedtyperepo.h> -#include "fieldinverter.h" -#include "urlfieldinverter.h" -#include "dictionary.h" -#include "ordereddocumentinserter.h" #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.documentinverter"); -namespace search -{ +LOG_SETUP(".memoryindex.documentinverter"); -namespace memoryindex -{ +namespace search::memoryindex { using document::Field; using document::FieldValue; @@ -208,8 +204,5 @@ DocumentInverter::pushDocuments(Dictionary &dict, } } - -} // namespace memoryindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h b/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h index 415271a0990..2e84b0421b1 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h @@ -69,7 +69,7 @@ private: * @return schema used by this index */ const index::Schema & - getSchema(void) const + getSchema() const { return _schema; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp b/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp index e6a214ec4aa..403c9f7da96 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp @@ -1,17 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.featurestore"); #include "featurestore.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/datastore/datastore.hpp> -namespace search -{ - -namespace memoryindex -{ +namespace search::memoryindex { constexpr size_t MIN_CLUSTERS = 1024u; @@ -105,7 +98,7 @@ FeatureStore::FeatureStore(const Schema &schema) } -FeatureStore::~FeatureStore(void) +FeatureStore::~FeatureStore() { _store.dropBuffers(); } @@ -155,8 +148,4 @@ FeatureStore::moveFeatures(uint32_t packedIndex, return moveFeatures(ref, bitLen); } - -} // namespace memoryindex - - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h b/searchlib/src/vespa/searchlib/memoryindex/featurestore.h index 651997d2dfb..84bf5078b33 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/featurestore.h @@ -97,7 +97,7 @@ public: */ FeatureStore(const Schema &schema); - ~FeatureStore(void); + ~FeatureStore(); /** * Add features to feature store @@ -233,7 +233,7 @@ public: } void - clearHoldLists(void) + clearHoldLists() { _store.clearHoldLists(); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp index 859aff99c0a..e635256a030 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fieldinverter.h" +#include "ordereddocumentinserter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/searchlib/util/url.h> #include <stdexcept> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> -#include "ordereddocumentinserter.h" #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/document/annotation/annotation.h> @@ -182,7 +181,7 @@ struct WordRefRadix { }; void -FieldInverter::sortWords(void) +FieldInverter::sortWords() { assert(_wordRefs.size() > 1); @@ -235,7 +234,7 @@ FieldInverter::startElement(int32_t weight) void -FieldInverter::endElement(void) +FieldInverter::endElement() { _elems.back().setLen(_wpos); _wpos = 0; diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h index db2a387a52a..029a6ceec57 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h @@ -215,7 +215,7 @@ public: * End an element. */ void - endElement(void); + endElement(); private: /** @@ -305,7 +305,7 @@ private: } void - stepWordPos(void) + stepWordPos() { ++_wpos; } @@ -330,7 +330,7 @@ private: * @return schema used by this index */ const index::Schema & - getSchema(void) const + getSchema() const { return _schema; } @@ -339,14 +339,14 @@ private: * Clear internal memory structures. */ void - reset(void); + reset(); /** * Calculate word numbers and replace word references with word * numbers in internal memory structures. */ void - sortWords(void); + sortWords(); void moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); diff --git a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h index aef68b62d23..0218f45529a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h @@ -2,21 +2,23 @@ #pragma once -namespace search -{ +#include <vespa/vespalib/stllike/string.h> +#include <cstdint> -namespace memoryindex -{ +namespace search { + +namespace index { class DocIdAndFeatures; } + +namespace memoryindex { /** * Interface class for ordered document inserter. * * Insert order must be properly sorted, by (word, docId) */ -class IOrderedDocumentInserter -{ +class IOrderedDocumentInserter { public: - virtual ~IOrderedDocumentInserter() { } + virtual ~IOrderedDocumentInserter() {} /** * Set next word to operate on. @@ -26,8 +28,7 @@ public: /** * Add (word, docId) tuple with given features. */ - virtual void add(uint32_t docId, - const index::DocIdAndFeatures &features) = 0; + virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; /** * Remove (word, docId) tuple. @@ -48,5 +49,4 @@ public: }; } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp index dcd4da4a8cf..13f5349d6ab 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp @@ -43,7 +43,7 @@ MemoryFieldIndex::MemoryFieldIndex(const Schema & schema, uint32_t fieldId) _inserter(std::make_unique<OrderedDocumentInserter>(*this)) { } -MemoryFieldIndex::~MemoryFieldIndex(void) +MemoryFieldIndex::~MemoryFieldIndex() { _postingListStore.disableFreeLists(); _postingListStore.disableElemHoldList(); @@ -96,7 +96,7 @@ MemoryFieldIndex::findFrozen(const vespalib::stringref word) const void -MemoryFieldIndex::compactFeatures(void) +MemoryFieldIndex::compactFeatures() { std::vector<uint32_t> toHold; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h index e00573374d0..862d72500c8 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h @@ -100,7 +100,7 @@ public: } MemoryFieldIndex(const index::Schema &schema, uint32_t fieldId); - ~MemoryFieldIndex(void); + ~MemoryFieldIndex(); PostingList::Iterator find(const vespalib::stringref word) const; PostingList::ConstIterator @@ -138,7 +138,7 @@ private: } void - incGeneration(void) + incGeneration() { _generationHandler.incGeneration(); } @@ -149,7 +149,7 @@ public: } void - compactFeatures(void); + compactFeatures(); void dump(search::index::IndexBuilder & indexBuilder); diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp index d65dbdf5910..e79032718af 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/stllike/string.h> +#include "ordereddocumentinserter.h" +#include "i_document_insert_listener.h" + #include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> -#include "ordereddocumentinserter.h" -#include "i_document_insert_listener.h" #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -17,22 +16,14 @@ #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btree.hpp> -LOG_SETUP(".memoryindex.orderedfieldinserter"); - -namespace search -{ - -namespace memoryindex -{ +namespace search::memoryindex { -namespace -{ +namespace { const vespalib::string emptyWord = ""; } - OrderedDocumentInserter::OrderedDocumentInserter(MemoryFieldIndex &fieldIndex) : _word(), _prevDocId(noDocId), @@ -152,7 +143,4 @@ OrderedDocumentInserter::getWordRef() const return _dItr.getKey()._wordRef; } - -} - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp index ab59718d064..117f69b7954 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "urlfieldinverter.h" #include "fieldinverter.h" #include <vespa/document/datatype/urldatatype.h> @@ -9,6 +8,7 @@ #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> #include <vespa/searchlib/common/sort.h> + #include <vespa/log/log.h> LOG_SETUP(".memoryindex.urlfieldinverter"); diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp b/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp index b3c6c0efb50..db76226c697 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".memoryindex.wordstore"); #include "wordstore.h" #include <vespa/searchlib/datastore/datastore.hpp> @@ -24,7 +21,7 @@ WordStore::WordStore() } -WordStore::~WordStore(void) +WordStore::~WordStore() { _store.dropBuffers(); } diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp index ec49a5dd195..591adf9188a 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp @@ -1,7 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/parsequery/parse.h> +#include "parse.h" #include <vespa/vespalib/objects/nbo.h> +#include <cassert> namespace search { @@ -74,7 +75,7 @@ ParseItem::ParseItem(ItemType type, const char *term) SetTerm(term); } -ParseItem::~ParseItem(void) +ParseItem::~ParseItem() { delete _next; delete _sibling; @@ -159,7 +160,7 @@ ParseItem::AppendBuffer(RawBuf *buf) const } size_t -ParseItem::GetBufferLen(void) const +ParseItem::GetBufferLen() const { // Calculate the length of the buffer. uint32_t indexLen = _indexName.size(); diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index 23fee8ea6f7..4f0f35d3151 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -216,7 +216,7 @@ public: */ void AppendBuffer(RawBuf *buf) const; - size_t GetBufferLen(void) const; + size_t GetBufferLen() const; }; } diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp index 784f85b95c4..61fdc4e4ff1 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp @@ -7,27 +7,26 @@ * Copyright (C) 2003 Overture Services Norway AS * ALL RIGHTS RESERVED */ -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "simplequerystack.h" #include <vespa/vespalib/util/vstringfmt.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> -#include <vespa/searchlib/parsequery/simplequerystack.h> +#include <vespa/log/log.h> LOG_SETUP(".search.simplequerystack"); using vespalib::make_vespa_string; namespace search { -SimpleQueryStack::SimpleQueryStack(void) +SimpleQueryStack::SimpleQueryStack() : _numItems(0), _stack(NULL), _FP_queryOK(true) { } -SimpleQueryStack::~SimpleQueryStack(void) +SimpleQueryStack::~SimpleQueryStack() { delete _stack; } @@ -49,7 +48,7 @@ SimpleQueryStack::Push(search::ParseItem *item) } search::ParseItem * -SimpleQueryStack::Pop(void) +SimpleQueryStack::Pop() { search::ParseItem *item = _stack; if (_stack != NULL) { @@ -69,7 +68,7 @@ SimpleQueryStack::AppendBuffer(search::RawBuf *buf) const } size_t -SimpleQueryStack::GetBufferLen(void) const +SimpleQueryStack::GetBufferLen() const { size_t result; @@ -83,13 +82,13 @@ SimpleQueryStack::GetBufferLen(void) const } uint32_t -SimpleQueryStack::GetSize(void) +SimpleQueryStack::GetSize() { return _numItems; } bool -SimpleQueryStack::_FP_isAllowed(void) +SimpleQueryStack::_FP_isAllowed() { return _FP_queryOK; } @@ -299,7 +298,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) vespalib::string key = ReadString(p); vespalib::string value = ReadString(p); uint64_t sub_queries = ReadUint64(p); - result.append(make_vespa_string("%s:%s:%" PRIx64, key.c_str(), value.c_str(), sub_queries)); + result.append(make_vespa_string("%s:%s:%lx", key.c_str(), value.c_str(), sub_queries)); if (i < feature_count - 1) { result.append(','); } @@ -311,7 +310,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) vespalib::string key = ReadString(p); uint64_t value = ReadUint64(p); uint64_t sub_queries = ReadUint64(p); - result.append(make_vespa_string("%s:%" PRIu64 ":%" PRIx64, key.c_str(), value, sub_queries)); + result.append(make_vespa_string("%s:%zu:%lx", key.c_str(), value, sub_queries)); if (i < range_feature_count - 1) { result.append(','); } diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h index fd6bced2704..01886e71c01 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h @@ -50,11 +50,11 @@ public: /** * Constructor for SimpleQueryStack. */ - SimpleQueryStack(void); + SimpleQueryStack(); /** * Destructor for SimpleQueryStack. */ - ~SimpleQueryStack(void); + ~SimpleQueryStack(); /** * Push an item on the stack. * @param item The search::ParseItem to push. @@ -64,12 +64,12 @@ public: * Pop an item of the stack. * @return Pointer to the search::ParseItem poped, or NULL if stack is empty. */ - search::ParseItem *Pop(void); + search::ParseItem *Pop(); /** * Top node of the stack. * @return Pointer to the top search::ParseItem, or NULL if stack is empty. */ - search::ParseItem *Top(void) { return _stack; } + search::ParseItem *Top() { return _stack; } /** * Encode the contents of the stack in a binary buffer. @@ -77,12 +77,12 @@ public: */ void AppendBuffer(search::RawBuf *buf) const; - size_t GetBufferLen(void) const; + size_t GetBufferLen() const; /** * Return the number of items on the stack. * @return The number of items on the stack. */ - uint32_t GetSize(void); + uint32_t GetSize(); /** * Set the number of items on the stack. * This can be used by QTs that change the stack @@ -95,7 +95,7 @@ public: * Is it possible to run this query on FirstPage? * @return true if ok */ - bool _FP_isAllowed(void); + bool _FP_isAllowed(); /** * Make a string representation of the search::RawBuf representing a querystack. * @param theBuf The querystack encoded buffer. diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index 17e1cc16baa..f406f8a70c9 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -6,11 +6,10 @@ * Copyright (C) 2003 Overture Services Norway AS * ALL RIGHTS RESERVED */ -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/parsequery/stackdumpiterator.h> -#include <vespa/searchlib/util/rawbuf.h> +#include "stackdumpiterator.h" #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> +#include <cassert> using search::query::PredicateQueryTerm; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index 4ccd7660a0c..fd510b3a904 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -93,25 +93,25 @@ public: * @return true if there is a new item, false if there are no more items * or if there was errors in extracting the next item. */ - bool next(void); + bool next(); /** * Get the number of the current item. * * @return The ordinal of the current item. -1 if at the start. */ - int getNum(void) const { return _currNum; } + int getNum() const { return _currNum; } /** * Get the type of the current item. * @return the type. */ - ParseItem::ItemType getType(void) const { return _currType; } + ParseItem::ItemType getType() const { return _currType; } /** * Get the type of the current item. * @return the type. */ - ParseItem::ItemCreator getCreator(void) const { return _currCreator; } + ParseItem::ItemCreator getCreator() const { return _currCreator; } /** * Get the rank weight of the current item. @@ -141,9 +141,9 @@ public: **/ uint32_t getFlags() const { return _currFlags; } - uint32_t getArity(void) const { return _currArity; } + uint32_t getArity() const { return _currArity; } - uint32_t getArg1(void) const { return _currArg1; } + uint32_t getArg1() const { return _currArg1; } double getArg2() const { return _currArg2; } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp index 7bb3a2abbcf..5970f565783 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "predicate_interval_store.h" #include "predicate_index.h" #include <vespa/searchlib/datastore/bufferstate.h> #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/searchlib/datastore/entryref.h> -#include <vespa/log/log.h> -LOG_SETUP(".predicate_interval_store"); using search::datastore::BufferState; using search::datastore::EntryRef; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.cpp index 16052ec41d0..c970882bffc 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "predicate_range_expander.h" + #include <vespa/log/log.h> LOG_SETUP(".predicate_range_expander"); diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index c6f7928f4d0..9b9c8eaaa3e 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -3,10 +3,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/exceptions.h> +#include <climits> -namespace search { -namespace predicate { +namespace search::predicate { /** * Helper class for expanding a point in a predicate range query to @@ -92,7 +91,4 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig } } - -} // namespace search::predicate -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_ref_cache.h b/searchlib/src/vespa/searchlib/predicate/predicate_ref_cache.h index 9aae296311a..7393cf40947 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_ref_cache.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_ref_cache.h @@ -4,6 +4,8 @@ #pragma once #include <set> +#include <cstdint> +#include <cassert> namespace search { namespace predicate { diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp index 41d754c6649..9823f984a1b 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp @@ -1,15 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "predicate_tree_analyzer.h" #include <vespa/document/predicate/predicate.h> #include <algorithm> #include <iostream> #include <cmath> -#include <vespa/log/log.h> - -LOG_SETUP(".predicate_tree_analyzer"); using document::Predicate; using std::map; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_zero_constraint_posting_list.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_zero_constraint_posting_list.cpp index 4ca88d5b4b8..d0803bd4d64 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_zero_constraint_posting_list.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_zero_constraint_posting_list.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - #include "predicate_zero_constraint_posting_list.h" -#include <vespa/log/log.h> -LOG_SETUP(".predicate_zero_constraint_posting_list"); namespace search { namespace predicate { diff --git a/searchlib/src/vespa/searchlib/query/base.cpp b/searchlib/src/vespa/searchlib/query/base.cpp index 772fcc6a6ec..1970817c9e3 100644 --- a/searchlib/src/vespa/searchlib/query/base.cpp +++ b/searchlib/src/vespa/searchlib/query/base.cpp @@ -3,7 +3,7 @@ namespace search { -Object::~Object(void) { } +Object::~Object() { } vespalib::string Object::toString() const { diff --git a/searchlib/src/vespa/searchlib/query/base.h b/searchlib/src/vespa/searchlib/query/base.h index b9d3a7cfc33..30efe779614 100644 --- a/searchlib/src/vespa/searchlib/query/base.h +++ b/searchlib/src/vespa/searchlib/query/base.h @@ -60,7 +60,7 @@ typedef std::vector<DocumentIdT> DocumentIdList; class Object { public: - virtual ~Object(void); + virtual ~Object(); /// Returns an allocated(new) object that is identical to this one. virtual Object * duplicate() const = 0; /// Gives you streamability of the object. Object does nothing. diff --git a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h index c0d47f0f11d..05f0c9033d5 100644 --- a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h +++ b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/vespalib/objects/cloneable.h> +#include <memory> namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp index 9c3dec5c1b2..c029dcdc13a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "booleanmatchiteratorwrapper.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/create-class-cpp.sh b/searchlib/src/vespa/searchlib/queryeval/create-class-cpp.sh index 728ab7a11ce..d25e2a84754 100755 --- a/searchlib/src/vespa/searchlib/queryeval/create-class-cpp.sh +++ b/searchlib/src/vespa/searchlib/queryeval/create-class-cpp.sh @@ -8,13 +8,9 @@ name=`echo $class | tr 'A-Z' 'a-z'` cat <<EOF // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP(".$name"); -#include <vespa/fastos/fastos.h> #include "$name.h" -namespace search { -namespace queryeval { +namespace search::queryeval { $class::$class() { @@ -24,6 +20,5 @@ $class::~$class() { } -} // namespace queryeval -} // namespace search +} EOF diff --git a/searchlib/src/vespa/searchlib/queryeval/document_weight_search_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/document_weight_search_iterator.cpp index 9f876e788bf..947ae6720f5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/document_weight_search_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/document_weight_search_iterator.cpp @@ -1,4 +1,3 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_weight_search_iterator.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 015b3f25499..7554883cce8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dot_product_blueprint.h" #include "dot_product_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp index b500e977848..475686d550e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "emptysearch.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp index 30ae5e023ef..070d4b80f0b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "equiv_blueprint.h" #include "equivsearch.h" #include <vespa/vespalib/objects/visit.hpp> diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp index 85946e1a758..cf8d81747e8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakesearch"); -#include <vespa/vespalib/objects/visit.h> #include "fake_search.h" #include <vespa/searchlib/fef/termfieldmatchdataposition.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/objects/visit.h> namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp index 60cdefab2c1..caa7c364b06 100644 --- a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fieldspec"); #include "field_spec.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h index 86c2d13ad8c..06834bd0fb8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h +++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h @@ -32,7 +32,7 @@ public: return _source.getFast(docId); } - uint32_t getDocIdLimit(void) const { + uint32_t getDocIdLimit() const { return _source.getCommittedDocIdLimit(); } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index b7e9cb8efc3..e39e27c0bf9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".queryeval.leaf_blueprints"); #include "leaf_blueprints.h" #include "emptysearch.h" #include "simplesearch.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp index 198634c56a9..d78e5ae2eb7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/monitoring_dump_iterator.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "monitoring_dump_iterator.h" #include <vespa/log/log.h> LOG_SETUP(".queryeval.monitoring_dump_iterator"); -#include "monitoring_dump_iterator.h" namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index 3ba06875441..41ebca00b09 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "multisearch.h" #include <vespa/vespalib/objects/visit.hpp> diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp index b33ab946acc..c39ed08982e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".nearsearch"); - #include "nearsearch.h" #include <vespa/vespalib/objects/visit.h> #include <vespa/vespalib/util/priority_queue.h> #include <limits> #include <set> +#include <vespa/log/log.h> +LOG_SETUP(".nearsearch"); + namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index cce2f22797b..449523b0da5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "simple_phrase_blueprint.h" #include "simple_phrase_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> #include <map> diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index 9a624724933..686bf3608e7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".simple_phrase_search"); - #include "simple_phrase_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp index f114eef8b39..20e3f482b62 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simpleresult.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".simpleresult"); #include "simpleresult.h" +#include <cassert> +#include <ostream> namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/simplesearch.cpp b/searchlib/src/vespa/searchlib/queryeval/simplesearch.cpp index 2d5b6b3e8b8..17641b38634 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simplesearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simplesearch.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".simplesearch"); #include "simplesearch.h" #include <vespa/vespalib/objects/visit.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/split_float.cpp b/searchlib/src/vespa/searchlib/queryeval/split_float.cpp index dd6a370ceda..f4a4c231871 100644 --- a/searchlib/src/vespa/searchlib/queryeval/split_float.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/split_float.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// $Id$ -#include <vespa/fastos/fastos.h> #include "split_float.h" -#include <ctype.h> +#include <cctype> namespace search { namespace queryeval { diff --git a/searchlib/src/vespa/searchlib/queryeval/truesearch.cpp b/searchlib/src/vespa/searchlib/queryeval/truesearch.cpp index 0c808ce06f2..bbc604db772 100644 --- a/searchlib/src/vespa/searchlib/queryeval/truesearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/truesearch.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "truesearch.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index d92ed9007bd..96ddaae1b03 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wand_parts.h" #include "parallel_weak_and_blueprint.h" #include "parallel_weak_and_search.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp index 05a2d7d6822..4250f424d32 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".queryeval.parallel_weak_and_search"); + #include "parallel_weak_and_search.h" +#include <vespa/searchlib/queryeval/document_weight_search_iterator.h> #include <vespa/searchlib/queryeval/monitoring_dump_iterator.h> -#include <vespa/searchlib/queryeval/monitoring_search_iterator.h> -#include "wand_parts.h" -#include <vespa/vespalib/objects/visit.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/fef/matchdatalayout.h> -#include <vespa/searchlib/queryeval/document_weight_search_iterator.h> +#include <vespa/vespalib/objects/visit.h> + +#include <vespa/log/log.h> +LOG_SETUP(".queryeval.parallel_weak_and_search"); using vespalib::make_string; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index 2fe27f8504b..af3e0b866f1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "wand_parts.h" #include "weak_and_search.h" #include <vespa/searchlib/queryeval/orsearch.h> diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index 7daec1159ca..d449ff3ee28 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "dense_tensor_attribute_saver.h" #include <vespa/searchlib/util/bufferwriter.h> #include "dense_tensor_store.h" diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp index 6c22434afe9..104f563afb3 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "generic_tensor_attribute_saver.h" -#include <vespa/searchlib/util/bufferwriter.h> #include "generic_tensor_store.h" +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_store.cpp index df22869f2d3..85befd85fe5 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_store.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "tensor_store.h" #include <vespa/searchlib/datastore/datastore.hpp> diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_store.h index dc8e8bab8d5..ea081d30067 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_store.h @@ -50,7 +50,7 @@ public: } void - clearHoldLists(void) + clearHoldLists() { _store.clearHoldLists(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/directory_handler.h b/searchlib/src/vespa/searchlib/test/directory_handler.h index 11d41e99809..2f2a6192813 100644 --- a/searchcore/src/vespa/searchcore/proton/test/directory_handler.h +++ b/searchlib/src/vespa/searchlib/test/directory_handler.h @@ -5,7 +5,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/string.h> -namespace proton { +namespace search { namespace test { class DirectoryHandler diff --git a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp index 1b4ec11f093..e8e41f61ae9 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/test/diskindex/testdiskindex.h> + +#include "testdiskindex.h" #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/io/fileutil.h> @@ -55,7 +55,7 @@ struct Builder void -TestDiskIndex::buildSchema(void) +TestDiskIndex::buildSchema() { _schema.addIndexField(Schema::IndexField("f1", DataType::STRING)); _schema.addIndexField(Schema::IndexField("f2", DataType::STRING)); diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp index 0b25d954e74..016b899c075 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp @@ -37,7 +37,7 @@ ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe) void -ThreeLevelCountWriteBuffers::flush(void) +ThreeLevelCountWriteBuffers::flush() { _ssFileBitSize = _sse.getWriteOffset(); _spFileBitSize = _spe.getWriteOffset(); diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h index a7fe87d83bf..09147e61c6a 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h @@ -29,7 +29,7 @@ public: ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe); void - flush(void); + flush(); // unit test method. Just pads without writing proper header void diff --git a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp index cfd06a86d0e..6efea89cd7f 100644 --- a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp +++ b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "document_weight_attribute_helper.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.cpp b/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.cpp index cda9314366c..57e61270913 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.cpp @@ -1,22 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".bitdecode64"); #include "bitencode64.h" #include "bitdecode64.h" - -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { template class BitDecode64<true>; - template class BitDecode64<false>; -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.h b/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.h index 05dfdaf0fb1..120a1efdedf 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/bitdecode64.h @@ -51,7 +51,7 @@ public: } uint64_t - getOffset(void) const + getOffset() const { return 64 * (_valI - _comprBase - 1) - this->_preRead - _bitOffsetBase; @@ -64,13 +64,13 @@ public: } const uint64_t * - getComprBase(void) const + getComprBase() const { return _comprBase; } int - getBitOffsetBase(void) const + getBitOffsetBase() const { return _bitOffsetBase; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.cpp b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.cpp index 9ceea95e01d..98cdb50c3fc 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.cpp @@ -1,19 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".bitencode64"); #include "bitencode64.h" - -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { template <bool bigEndian> -BitEncode64<bigEndian>::BitEncode64(void) +BitEncode64<bigEndian>::BitEncode64() : bitcompression::EncodeContext64<bigEndian>(), _cbuf(*this) { @@ -21,17 +13,12 @@ BitEncode64<bigEndian>::BitEncode64(void) this->afterWrite(_cbuf, 0, 0); } - template <bool bigEndian> -BitEncode64<bigEndian>::~BitEncode64(void) +BitEncode64<bigEndian>::~BitEncode64() { } template class BitEncode64<true>; - template class BitEncode64<false>; - -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h index 893410aa86b..f1bfaab68a6 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h @@ -17,20 +17,20 @@ class BitEncode64 : public bitcompression::EncodeContext64<bigEndian> search::ComprFileWriteContext _cbuf; public: - BitEncode64(void); + BitEncode64(); - ~BitEncode64(void); + ~BitEncode64(); typedef bitcompression::EncodeContext64<bigEndian> EC; void - writeComprBuffer(void) + writeComprBuffer() { _cbuf.writeComprBuffer(true); } void - writeComprBufferIfNeeded(void) + writeComprBufferIfNeeded() { if (this->_valI >= this->_valE) _cbuf.writeComprBuffer(false); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index 2b7b5398e1e..fcc83b39138 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakeegcompr64filterocc"); #include "fakeegcompr64filterocc.h" +#include "fpfactory.h" #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include "fpfactory.h" using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataPosition; @@ -16,11 +13,7 @@ using search::fef::TermFieldMatchDataPosition; #include "bitencode64.h" #include "bitdecode64.h" -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { #define DEBUG_EGCOMPR64FILTEROCC_PRINTF 0 #define DEBUG_EGCOMPR64FILTEROCC_ASSERT 1 @@ -412,7 +405,7 @@ setupT(const FakeWord &fw) } -FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc(void) +FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc() { free(_compressedMalloc); free(_l1SkipCompressedMalloc); @@ -423,69 +416,69 @@ FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc(void) void -FakeEGCompr64FilterOcc::forceLink(void) +FakeEGCompr64FilterOcc::forceLink() { } size_t -FakeEGCompr64FilterOcc::bitSize(void) const +FakeEGCompr64FilterOcc::bitSize() const { return _bitSize; } bool -FakeEGCompr64FilterOcc::hasWordPositions(void) const +FakeEGCompr64FilterOcc::hasWordPositions() const { return false; } size_t -FakeEGCompr64FilterOcc::skipBitSize(void) const +FakeEGCompr64FilterOcc::skipBitSize() const { return _l1SkipBitSize + _l2SkipBitSize + _l3SkipBitSize + _l4SkipBitSize; } size_t -FakeEGCompr64FilterOcc::l1SkipBitSize(void) const +FakeEGCompr64FilterOcc::l1SkipBitSize() const { return _l1SkipBitSize; } size_t -FakeEGCompr64FilterOcc::l2SkipBitSize(void) const +FakeEGCompr64FilterOcc::l2SkipBitSize() const { return _l2SkipBitSize; } size_t -FakeEGCompr64FilterOcc::l3SkipBitSize(void) const +FakeEGCompr64FilterOcc::l3SkipBitSize() const { return _l3SkipBitSize; } size_t -FakeEGCompr64FilterOcc::l4SkipBitSize(void) const +FakeEGCompr64FilterOcc::l4SkipBitSize() const { return _l4SkipBitSize; } int -FakeEGCompr64FilterOcc::lowLevelSinglePostingScan(void) const +FakeEGCompr64FilterOcc::lowLevelSinglePostingScan() const { return 0; } int -FakeEGCompr64FilterOcc::lowLevelSinglePostingScanUnpack(void) const +FakeEGCompr64FilterOcc::lowLevelSinglePostingScanUnpack() const { return 0; } @@ -559,7 +552,7 @@ public: } uint32_t - getDocIdDelta(void) + getDocIdDelta() { uint32_t ret; unsigned int length; @@ -571,7 +564,7 @@ public: } uint32_t - getL1SkipDocIdDelta(void) + getL1SkipDocIdDelta() { uint32_t ret; unsigned int length; @@ -583,7 +576,7 @@ public: } uint32_t - getL2SkipDocIdDelta(void) + getL2SkipDocIdDelta() { uint32_t ret; unsigned int length; @@ -595,7 +588,7 @@ public: } uint32_t - getL3SkipDocIdDelta(void) + getL3SkipDocIdDelta() { uint32_t ret; unsigned int length; @@ -631,7 +624,7 @@ public: uint32_t lastDocId, const search::fef::TermFieldMatchDataArray &matchData); - ~FakeFilterOccEGCompressed64ArrayIterator(void); + ~FakeFilterOccEGCompressed64ArrayIterator(); void doUnpack(uint32_t docId) override; void doSeek(uint32_t docId) override; @@ -688,7 +681,7 @@ initRange(uint32_t begin, uint32_t end) template <bool bigEndian> FakeFilterOccEGCompressed64ArrayIterator<bigEndian>:: -~FakeFilterOccEGCompressed64ArrayIterator(void) +~FakeFilterOccEGCompressed64ArrayIterator() { } @@ -763,7 +756,7 @@ class FakeEGCompr64LEFilterOcc : public FakeEGCompr64FilterOcc public: FakeEGCompr64LEFilterOcc(const FakeWord &fw); - ~FakeEGCompr64LEFilterOcc(void); + ~FakeEGCompr64LEFilterOcc(); }; @@ -773,7 +766,7 @@ FakeEGCompr64LEFilterOcc::FakeEGCompr64LEFilterOcc(const FakeWord &fw) } -FakeEGCompr64LEFilterOcc::~FakeEGCompr64LEFilterOcc(void) +FakeEGCompr64LEFilterOcc::~FakeEGCompr64LEFilterOcc() { } @@ -818,7 +811,7 @@ FakeEGCompr64SkipFilterOcc<false>::FakeEGCompr64SkipFilterOcc(const FakeWord &fw template <bool doSkip> -FakeEGCompr64SkipFilterOcc<doSkip>::~FakeEGCompr64SkipFilterOcc(void) +FakeEGCompr64SkipFilterOcc<doSkip>::~FakeEGCompr64SkipFilterOcc() { } @@ -873,7 +866,7 @@ public: const std::string &name, const fef::TermFieldMatchDataArray &matchData); - ~FakeFilterOccEGCompressed64SkipArrayIterator(void); + ~FakeFilterOccEGCompressed64SkipArrayIterator(); void doL4SkipSeek(uint32_t docid); @@ -1003,7 +996,7 @@ initRange(uint32_t begin, uint32_t end) template <bool doSkip> FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>:: -~FakeFilterOccEGCompressed64SkipArrayIterator(void) +~FakeFilterOccEGCompressed64SkipArrayIterator() { } @@ -1512,7 +1505,4 @@ createIterator(const fef::TermFieldMatchDataArray &matchData) const matchData); } - -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp index 749803cc6ed..1c4b318ed71 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp @@ -1,20 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakefilterocc"); -#include <vespa/searchlib/queryeval/iterators.h> #include "fakefilterocc.h" #include "fpfactory.h" +#include <vespa/searchlib/queryeval/iterators.h> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataPosition; -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { static FPFactoryInit init(std::make_pair("FilterOcc", @@ -44,40 +37,40 @@ FakeFilterOcc::FakeFilterOcc(const FakeWord &fw) } -FakeFilterOcc::~FakeFilterOcc(void) +FakeFilterOcc::~FakeFilterOcc() { } void -FakeFilterOcc::forceLink(void) +FakeFilterOcc::forceLink() { } size_t -FakeFilterOcc::bitSize(void) const +FakeFilterOcc::bitSize() const { return 32 * _uncompressed.size(); } bool -FakeFilterOcc::hasWordPositions(void) const +FakeFilterOcc::hasWordPositions() const { return false; } int -FakeFilterOcc::lowLevelSinglePostingScan(void) const +FakeFilterOcc::lowLevelSinglePostingScan() const { return 0; } int -FakeFilterOcc::lowLevelSinglePostingScanUnpack(void) const +FakeFilterOcc::lowLevelSinglePostingScanUnpack() const { return 0; } @@ -116,7 +109,7 @@ public: const uint32_t *arrEnd, const fef::TermFieldMatchDataArray &matchData); - ~FakeFilterOccArrayIterator(void); + ~FakeFilterOccArrayIterator(); void doUnpack(uint32_t docId) override; void doSeek(uint32_t docId) override; @@ -175,7 +168,7 @@ FakeFilterOccArrayIterator::initRange(uint32_t begin, uint32_t end) } -FakeFilterOccArrayIterator::~FakeFilterOccArrayIterator(void) +FakeFilterOccArrayIterator::~FakeFilterOccArrayIterator() { } @@ -201,6 +194,4 @@ createIterator(const fef::TermFieldMatchDataArray &matchData) const matchData); } -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h index 2946f15c539..3110d9b3332 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h @@ -25,9 +25,9 @@ public: static void forceLink(); size_t bitSize() const override; - bool hasWordPositions(void) const override; - int lowLevelSinglePostingScan(void) const override; - int lowLevelSinglePostingScanUnpack(void) const override; + bool hasWordPositions() const override; + int lowLevelSinglePostingScan() const override; + int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; queryeval::SearchIterator * createIterator(const fef::TermFieldMatchDataArray &matchData) const override; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 01bd2551989..750be8b84dc 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -1,27 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakememtreeocc"); -#include <vespa/searchlib/queryeval/iterators.h> #include "fakememtreeocc.h" +#include "fpfactory.h" +#include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> #include <vespa/searchlib/memoryindex/postingiterator.h> -#include "fpfactory.h" #include <vespa/searchlib/util/postingpriorityqueue.h> +#include <vespa/log/log.h> +LOG_SETUP(".fakememtreeocc"); using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataPosition; -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { static FPFactoryInit init(std::make_pair("MemTreeOcc", @@ -72,40 +67,40 @@ FakeMemTreeOcc::FakeMemTreeOcc(const FakeWord &fw, } -FakeMemTreeOcc::~FakeMemTreeOcc(void) +FakeMemTreeOcc::~FakeMemTreeOcc() { } void -FakeMemTreeOcc::forceLink(void) +FakeMemTreeOcc::forceLink() { } size_t -FakeMemTreeOcc::bitSize(void) const +FakeMemTreeOcc::bitSize() const { return _tree.bitSize(_allocator) + _featureBitSize; } bool -FakeMemTreeOcc::hasWordPositions(void) const +FakeMemTreeOcc::hasWordPositions() const { return true; } int -FakeMemTreeOcc::lowLevelSinglePostingScan(void) const +FakeMemTreeOcc::lowLevelSinglePostingScan() const { return 0; } int -FakeMemTreeOcc::lowLevelSinglePostingScanUnpack(void) const +FakeMemTreeOcc::lowLevelSinglePostingScanUnpack() const { return 0; } @@ -152,7 +147,7 @@ FakeMemTreeOccMgr::FakeMemTreeOccMgr(const Schema &schema) } -FakeMemTreeOccMgr::~FakeMemTreeOccMgr(void) +FakeMemTreeOccMgr::~FakeMemTreeOccMgr() { std::vector<std::shared_ptr<PostingIdx> >::iterator it(_postingIdxs.begin()); @@ -167,34 +162,34 @@ FakeMemTreeOccMgr::~FakeMemTreeOccMgr(void) void -FakeMemTreeOccMgr::freeze(void) +FakeMemTreeOccMgr::freeze() { _allocator.freeze(); } void -FakeMemTreeOccMgr::transferHoldLists(void) +FakeMemTreeOccMgr::transferHoldLists() { _allocator.transferHoldLists(_generationHandler.getCurrentGeneration()); } void -FakeMemTreeOccMgr::incGeneration(void) +FakeMemTreeOccMgr::incGeneration() { _generationHandler.incGeneration(); } void -FakeMemTreeOccMgr::trimHoldLists(void) +FakeMemTreeOccMgr::trimHoldLists() { _allocator.trimHoldLists(_generationHandler.getFirstUsedGeneration()); } void -FakeMemTreeOccMgr::sync(void) +FakeMemTreeOccMgr::sync() { freeze(); transferHoldLists(); @@ -233,7 +228,7 @@ FakeMemTreeOccMgr::remove(uint32_t wordIdx, uint32_t docId) void -FakeMemTreeOccMgr::sortUnflushed(void) +FakeMemTreeOccMgr::sortUnflushed() { typedef std::vector<PendingOp>::iterator I; uint32_t seq = 0; @@ -245,7 +240,7 @@ FakeMemTreeOccMgr::sortUnflushed(void) void -FakeMemTreeOccMgr::flush(void) +FakeMemTreeOccMgr::flush() { typedef FeatureStore::RefType RefType; typedef std::vector<PendingOp>::iterator I; @@ -286,7 +281,7 @@ FakeMemTreeOccMgr::flush(void) } void -FakeMemTreeOccMgr::compactTrees(void) +FakeMemTreeOccMgr::compactTrees() { // compact full trees by calling incremental compaction methods in a loop @@ -298,7 +293,6 @@ FakeMemTreeOccMgr::compactTrees(void) itr.begin(); tree.setRoot(itr.moveFirstLeafNode(tree.getRoot()), _allocator); while (itr.valid()) { - // LOG(info, "Leaf moved to %d", UNWRAP(itr.getKey())); itr.moveNextLeafNode(); } } @@ -307,7 +301,7 @@ FakeMemTreeOccMgr::compactTrees(void) } void -FakeMemTreeOccMgr::finalize(void) +FakeMemTreeOccMgr::finalize() { flush(); } @@ -319,7 +313,7 @@ FakeMemTreeOccFactory::FakeMemTreeOccFactory(const Schema &schema) } -FakeMemTreeOccFactory::~FakeMemTreeOccFactory(void) +FakeMemTreeOccFactory::~FakeMemTreeOccFactory() { } @@ -389,7 +383,7 @@ FakeMemTreeOcc2Factory::FakeMemTreeOcc2Factory(const Schema &schema) } -FakeMemTreeOcc2Factory::~FakeMemTreeOcc2Factory(void) +FakeMemTreeOcc2Factory::~FakeMemTreeOcc2Factory() { } @@ -424,7 +418,4 @@ FakeMemTreeOcc2Factory::setup(const std::vector<const FakeWord *> &fws) LOG(info, "done compacting trees"); } - -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index acc28217d09..f3337d87239 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -69,9 +69,9 @@ public: {} void setSeq(uint32_t seq) { _seq = seq; } - uint32_t getWordIdx(void) const { return _wordIdx; } - uint32_t getDocId(void) const { return _docId; } - EntryRef getFeatureRef(void) const { return _features; } + uint32_t getWordIdx() const { return _wordIdx; } + uint32_t getDocId() const { return _docId; } + EntryRef getFeatureRef() const { return _features; } bool getRemove() const { return _removal; } bool operator<(const PendingOp &rhs) const { diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.cpp index 8d0915d4966..b68fac37fcd 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.cpp @@ -1,15 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakeposting"); #include "fakeposting.h" -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { FakePosting::FakePosting(const std::string &name) : _name(name) @@ -17,45 +10,43 @@ FakePosting::FakePosting(const std::string &name) } -FakePosting::~FakePosting(void) +FakePosting::~FakePosting() { } size_t -FakePosting::skipBitSize(void) const +FakePosting::skipBitSize() const { return l1SkipBitSize() + l2SkipBitSize() + l3SkipBitSize() + l4SkipBitSize(); } size_t -FakePosting::l1SkipBitSize(void) const +FakePosting::l1SkipBitSize() const { return 0; } size_t -FakePosting::l2SkipBitSize(void) const +FakePosting::l2SkipBitSize() const { return 0; } size_t -FakePosting::l3SkipBitSize(void) const +FakePosting::l3SkipBitSize() const { return 0; } size_t -FakePosting::l4SkipBitSize(void) const +FakePosting::l4SkipBitSize() const { return 0; } -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h index 946d1e05379..9958b21abc7 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h @@ -36,43 +36,43 @@ public: FakePosting(const std::string &name); - virtual ~FakePosting(void); + virtual ~FakePosting(); /* * Size of posting list, in bits. */ virtual size_t - bitSize(void) const = 0; + bitSize() const = 0; virtual size_t - skipBitSize(void) const; + skipBitSize() const; virtual size_t - l1SkipBitSize(void) const; + l1SkipBitSize() const; virtual size_t - l2SkipBitSize(void) const; + l2SkipBitSize() const; virtual size_t - l3SkipBitSize(void) const; + l3SkipBitSize() const; virtual size_t - l4SkipBitSize(void) const; + l4SkipBitSize() const; virtual bool - hasWordPositions(void) const = 0; + hasWordPositions() const = 0; /* * Single posting list performance, without feature unpack. */ virtual int - lowLevelSinglePostingScan(void) const = 0; + lowLevelSinglePostingScan() const = 0; /* * Single posting list performance, with feature unpack. */ virtual int - lowLevelSinglePostingScanUnpack(void) const = 0; + lowLevelSinglePostingScanUnpack() const = 0; /* * Two posting lists performance (same format) without feature unpack. @@ -93,7 +93,7 @@ public: virtual search::queryeval::SearchIterator * createIterator(const fef::TermFieldMatchDataArray &matchData) const = 0; - const std::string &getName(void) const + const std::string &getName() const { return _name; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index 5ad3140b5b2..d28d4e80fd8 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakeword"); #include "fakeword.h" #include <vespa/searchlib/index/postinglistfile.h> @@ -95,7 +92,7 @@ fillcorrelatedbitset(search::BitVector &bitvector, } -FakeWord::DocWordPosFeature::DocWordPosFeature(void) +FakeWord::DocWordPosFeature::DocWordPosFeature() : _elementId(0), _wordPos(0), _elementWeight(1), @@ -104,22 +101,22 @@ FakeWord::DocWordPosFeature::DocWordPosFeature(void) } -FakeWord::DocWordPosFeature::~DocWordPosFeature(void) +FakeWord::DocWordPosFeature::~DocWordPosFeature() { } -FakeWord::DocWordCollapsedFeature::DocWordCollapsedFeature(void) +FakeWord::DocWordCollapsedFeature::DocWordCollapsedFeature() { } -FakeWord::DocWordCollapsedFeature::~DocWordCollapsedFeature(void) +FakeWord::DocWordCollapsedFeature::~DocWordCollapsedFeature() { } -FakeWord::DocWordFeature::DocWordFeature(void) +FakeWord::DocWordFeature::DocWordFeature() : _docId(0), _collapsedDocWordFeatures(), _positions(0), @@ -127,7 +124,7 @@ FakeWord::DocWordFeature::DocWordFeature(void) { } -FakeWord::DocWordFeature::~DocWordFeature(void) +FakeWord::DocWordFeature::~DocWordFeature() { } @@ -208,7 +205,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, } -FakeWord::~FakeWord(void) +FakeWord::~FakeWord() { } @@ -609,7 +606,7 @@ FakeWord::validate(std::shared_ptr<FieldReader> &fieldReader, features._wordPositions.begin(); TermFieldMatchData *tfmd = matchData[0]; - LOG_ASSERT(tfmd != 0); + assert(tfmd != 0); tfmd->reset(features._docId); uint32_t elementResidue = features._elements.size(); @@ -752,7 +749,7 @@ FakeWord::dump(std::shared_ptr<FieldWriter> &fieldWriter, } -FakeWord::RandomizedReader::RandomizedReader(void) +FakeWord::RandomizedReader::RandomizedReader() : _r(), _fw(NULL), _wordIdx(0u), @@ -764,7 +761,7 @@ FakeWord::RandomizedReader::RandomizedReader(void) void -FakeWord::RandomizedReader::read(void) +FakeWord::RandomizedReader::read() { if (_ri != _re) { _r = *_ri; @@ -786,7 +783,7 @@ FakeWord::RandomizedReader::setup(const FakeWord *fw, } -FakeWord::RandomizedWriter::~RandomizedWriter(void) +FakeWord::RandomizedWriter::~RandomizedWriter() { } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index 8172fc68a43..b3b5f80a4a2 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "fakewordset.h" #include "fakeword.h" -#include <vespa/searchlib/index/schemautil.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".fakewordset"); -namespace search { - -namespace fakedata { +namespace search::fakedata { using index::PostingListParams; using index::SchemaUtil; @@ -33,7 +31,7 @@ applyDocIdBiasToVector(std::vector<FakeWord *> &v, uint32_t docIdBias) } -FakeWordSet::FakeWordSet(void) +FakeWordSet::FakeWordSet() : _words(NUM_WORDCLASSES), _schema(), _fieldsParams() @@ -52,7 +50,7 @@ FakeWordSet::FakeWordSet(bool hasElements, } -FakeWordSet::~FakeWordSet(void) +FakeWordSet::~FakeWordSet() { dropWords(); } @@ -130,7 +128,7 @@ FakeWordSet::setupWords(search::Rand48 &rnd, void -FakeWordSet::dropWords(void) +FakeWordSet::dropWords() { for (unsigned int i = 0; i < _words.size(); ++i) clearFakeWordVector(_words[i]); @@ -138,7 +136,7 @@ FakeWordSet::dropWords(void) int -FakeWordSet::getNumWords(void) +FakeWordSet::getNumWords() { int ret = 0; for (unsigned int i = 0; i < _words.size(); ++i) @@ -146,7 +144,6 @@ FakeWordSet::getNumWords(void) return ret; } - void FakeWordSet::addDocIdBias(uint32_t docIdBias) { @@ -154,7 +151,4 @@ FakeWordSet::addDocIdBias(uint32_t docIdBias) applyDocIdBiasToVector(_words[i], docIdBias); } - -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 51e87ffd817..a713a0e6aec 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -35,12 +35,12 @@ public: Schema _schema; std::vector<PosOccFieldsParams> _fieldsParams; - FakeWordSet(void); + FakeWordSet(); FakeWordSet(bool hasElements, bool hasElementWeights); - ~FakeWordSet(void); + ~FakeWordSet(); void setupParams(bool hasElements, @@ -53,31 +53,31 @@ public: unsigned int numWordsPerWordClass); void - dropWords(void); + dropWords(); int - getNumWords(void); + getNumWords(); const PosOccFieldsParams & - getFieldsParams(void) const + getFieldsParams() const { return _fieldsParams.back(); } uint32_t - getPackedIndex(void) const + getPackedIndex() const { return _fieldsParams.size() - 1; } const std::vector<PosOccFieldsParams> & - getAllFieldsParams(void) const + getAllFieldsParams() const { return _fieldsParams; } const Schema & - getSchema(void) const + getSchema() const { return _schema; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp index b1539e2ea2d..6b1cee01cad 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fakezcbfilterocc"); #include "fakezcbfilterocc.h" #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -12,11 +9,7 @@ LOG_SETUP(".fakezcbfilterocc"); using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataPosition; -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { static FPFactoryInit init(std::make_pair("ZcbFilterOcc", @@ -102,40 +95,40 @@ FakeZcbFilterOcc::FakeZcbFilterOcc(const FakeWord &fw) } -FakeZcbFilterOcc::~FakeZcbFilterOcc(void) +FakeZcbFilterOcc::~FakeZcbFilterOcc() { } void -FakeZcbFilterOcc::forceLink(void) +FakeZcbFilterOcc::forceLink() { } size_t -FakeZcbFilterOcc::bitSize(void) const +FakeZcbFilterOcc::bitSize() const { // Do not count the 3 padding bytes here. return 8 * (_compressed.size() - 3) ; } bool -FakeZcbFilterOcc::hasWordPositions(void) const +FakeZcbFilterOcc::hasWordPositions() const { return false; } int -FakeZcbFilterOcc::lowLevelSinglePostingScan(void) const +FakeZcbFilterOcc::lowLevelSinglePostingScan() const { return 0; } int -FakeZcbFilterOcc::lowLevelSinglePostingScanUnpack(void) const +FakeZcbFilterOcc::lowLevelSinglePostingScanUnpack() const { return 0; } @@ -178,7 +171,7 @@ public: unsigned int residue, const fef::TermFieldMatchDataArray &matchData); - ~FakeFilterOccZCBArrayIterator(void); + ~FakeFilterOccZCBArrayIterator(); void doUnpack(uint32_t docId) override; void doSeek(uint32_t docId) override; @@ -212,8 +205,7 @@ FakeFilterOccZCBArrayIterator::initRange(uint32_t begin, uint32_t end) } -FakeFilterOccZCBArrayIterator:: -~FakeFilterOccZCBArrayIterator(void) +FakeFilterOccZCBArrayIterator::~FakeFilterOccZCBArrayIterator() { } @@ -263,6 +255,4 @@ createIterator(const fef::TermFieldMatchDataArray &matchData) const matchData); } -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index ee9bed6ba02..9480d5ecd2e 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -3,6 +3,7 @@ #include "fakezcfilterocc.h" #include "fpfactory.h" #include <vespa/searchlib/diskindex/zcposocciterators.h> +#include <vespa/searchlib/diskindex/zcbuf.h> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; @@ -35,18 +36,16 @@ static FPFactoryInit init(std::make_pair("ZcFilterOcc", makeFPFactory<FPFactoryT<FakeZcFilterOcc> >)); + +template <typename EC> static void -zcEncode(std::vector<uint8_t> &bytes, - uint32_t num) +writeZcBuf(EC &e, ZcBuf &buf) { - for (;;) { - if (num < (1 << 7)) { - bytes.push_back(num); - break; - } - bytes.push_back((num & ((1 << 7) - 1)) | (1 << 7)); - num >>= 7; - } + uint32_t size = buf.size(); + uint8_t *bytes = buf._mallocStart; + uint32_t bytesOffset = reinterpret_cast<unsigned long>(bytes) & 7; + e.writeBits(reinterpret_cast<const uint64_t *>(bytes - bytesOffset), + bytesOffset * 8, size * 8); } #define ZCDECODE(valI, resop) \ @@ -138,11 +137,11 @@ void FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, bool dynamicK) { - std::vector<uint8_t> bytes; - std::vector<uint8_t> l1SkipBytes; - std::vector<uint8_t> l2SkipBytes; - std::vector<uint8_t> l3SkipBytes; - std::vector<uint8_t> l4SkipBytes; + ZcBuf bytes; + ZcBuf l1SkipBytes; + ZcBuf l2SkipBytes; + ZcBuf l3SkipBytes; + ZcBuf l4SkipBytes; uint32_t lastDocId = 0u; uint32_t lastL1SkipDocId = 0u; uint64_t lastL1SkipDocIdPos = 0; @@ -187,17 +186,23 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, fctx.allocComprBuf(64, 1); f.afterWrite(fctx, 0, 0); + // Ensure that some space is initially available in encoding buffers + bytes.maybeExpand(); + l1SkipBytes.maybeExpand(); + l2SkipBytes.maybeExpand(); + l3SkipBytes.maybeExpand(); + l4SkipBytes.maybeExpand(); while (d != de) { if (l1SkipCnt >= L1SKIPSTRIDE) { uint32_t docIdDelta = lastDocId - lastL1SkipDocId; assert(static_cast<int32_t>(docIdDelta) > 0); - zcEncode(l1SkipBytes, docIdDelta - 1); + l1SkipBytes.encode(docIdDelta - 1); uint64_t lastDocIdPos = bytes.size(); uint32_t docIdPosDelta = lastDocIdPos - lastL1SkipDocIdPos; - zcEncode(l1SkipBytes, docIdPosDelta - 1); + l1SkipBytes.encode(docIdPosDelta - 1); if (doFeatures) { featurePos = f.getWriteOffset(); - zcEncode(l1SkipBytes, featurePos - lastL1SkipFeaturePos - 1); + l1SkipBytes.encode(featurePos - lastL1SkipFeaturePos - 1); lastL1SkipFeaturePos = featurePos; } #if DEBUG_ZCFILTEROCC_PRINTF @@ -214,14 +219,13 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, docIdPosDelta = lastDocIdPos - lastL2SkipDocIdPos; uint64_t lastL1SkipPos = l1SkipBytes.size(); uint32_t l1SkipPosDelta = lastL1SkipPos - lastL2SkipL1SkipPos; - zcEncode(l2SkipBytes, docIdDelta - 1); - zcEncode(l2SkipBytes, docIdPosDelta - 1); + l2SkipBytes.encode(docIdDelta - 1); + l2SkipBytes.encode(docIdPosDelta - 1); if (doFeatures) { - zcEncode(l2SkipBytes, - featurePos - lastL2SkipFeaturePos - 1); + l2SkipBytes.encode(featurePos - lastL2SkipFeaturePos - 1); lastL2SkipFeaturePos = featurePos; } - zcEncode(l2SkipBytes, l1SkipPosDelta - 1); + l2SkipBytes.encode(l1SkipPosDelta - 1); #if DEBUG_ZCFILTEROCC_PRINTF printf("L2Encode docId=%d (+%d), docIdPos=%d (+%u)," " l1SkipPos=%d (+%u)\n", @@ -241,15 +245,14 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, uint64_t lastL2SkipPos = l2SkipBytes.size(); uint32_t l2SkipPosDelta = lastL2SkipPos - lastL3SkipL2SkipPos; - zcEncode(l3SkipBytes, docIdDelta - 1); - zcEncode(l3SkipBytes, docIdPosDelta - 1); + l3SkipBytes.encode(docIdDelta - 1); + l3SkipBytes.encode(docIdPosDelta - 1); if (doFeatures) { - zcEncode(l3SkipBytes, - featurePos - lastL3SkipFeaturePos - 1); + l3SkipBytes.encode(featurePos - lastL3SkipFeaturePos - 1); lastL3SkipFeaturePos = featurePos; } - zcEncode(l3SkipBytes, l1SkipPosDelta - 1); - zcEncode(l3SkipBytes, l2SkipPosDelta - 1); + l3SkipBytes.encode(l1SkipPosDelta - 1); + l3SkipBytes.encode(l2SkipPosDelta - 1); #if DEBUG_ZCFILTEROCC_PRINTF printf("L3Encode docId=%d (+%d), docIdPos=%d (+%u)," " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)\n", @@ -272,16 +275,15 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, uint64_t lastL3SkipPos = l3SkipBytes.size(); uint32_t l3SkipPosDelta = lastL3SkipPos - lastL4SkipL3SkipPos; - zcEncode(l4SkipBytes, docIdDelta - 1); - zcEncode(l4SkipBytes, docIdPosDelta - 1); + l4SkipBytes.encode(docIdDelta - 1); + l4SkipBytes.encode(docIdPosDelta - 1); if (doFeatures) { - zcEncode(l4SkipBytes, - featurePos - lastL4SkipFeaturePos - 1); + l4SkipBytes.encode(featurePos - lastL4SkipFeaturePos - 1); lastL4SkipFeaturePos = featurePos; } - zcEncode(l4SkipBytes, l1SkipPosDelta - 1); - zcEncode(l4SkipBytes, l2SkipPosDelta - 1); - zcEncode(l4SkipBytes, l3SkipPosDelta - 1); + l4SkipBytes.encode(l1SkipPosDelta - 1); + l4SkipBytes.encode(l2SkipPosDelta - 1); + l4SkipBytes.encode(l3SkipPosDelta - 1); #if DEBUG_ZCFILTEROCC_PRINTF printf("L4Encode docId=%d (+%d), docIdPos=%d (+%u)," " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)" @@ -303,14 +305,14 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, } } if (lastDocId == 0u) { - zcEncode(bytes, d->_docId - 1); + bytes.encode(d->_docId - 1); #if DEBUG_ZCFILTEROCC_PRINTF printf("Encode docId=%d\n", d->_docId); #endif } else { uint32_t docIdDelta = d->_docId - lastDocId; - zcEncode(bytes, docIdDelta - 1); + bytes.encode(docIdDelta - 1); #if DEBUG_ZCFILTEROCC_PRINTF printf("Encode docId=%d (+%d)\n", d->_docId, docIdDelta); @@ -353,22 +355,22 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, if (l1SkipBytes.size() > 0) { uint32_t docIdDelta = lastDocId - lastL1SkipDocId; assert(static_cast<int32_t>(docIdDelta) > 0); - zcEncode(l1SkipBytes, docIdDelta - 1); + l1SkipBytes.encode(docIdDelta - 1); } if (l2SkipBytes.size() > 0) { uint32_t docIdDelta = lastDocId - lastL2SkipDocId; assert(static_cast<int32_t>(docIdDelta) > 0); - zcEncode(l2SkipBytes, docIdDelta - 1); + l2SkipBytes.encode(docIdDelta - 1); } if (l3SkipBytes.size() > 0) { uint32_t docIdDelta = lastDocId - lastL3SkipDocId; assert(static_cast<int32_t>(docIdDelta) > 0); - zcEncode(l3SkipBytes, docIdDelta - 1); + l3SkipBytes.encode(docIdDelta - 1); } if (l4SkipBytes.size() > 0) { uint32_t docIdDelta = lastDocId - lastL4SkipDocId; assert(static_cast<int32_t>(docIdDelta) > 0); - zcEncode(l4SkipBytes, docIdDelta - 1); + l4SkipBytes.encode(docIdDelta - 1); } _hitDocs = fw._postings.size(); _docIdLimit = fw._docIdLimit; @@ -419,45 +421,16 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, e.writeBits(0, bytePad); size_t docIdSize = bytes.size(); if (docIdSize > 0) { - uint8_t *docIdBytes = &bytes[0]; - uint32_t docIdBytesOffset = - reinterpret_cast<unsigned long>(docIdBytes) & 7; - e.writeBits(reinterpret_cast<const uint64_t *>(docIdBytes - - docIdBytesOffset), - docIdBytesOffset * 8, - docIdSize * 8); + writeZcBuf(e, bytes); } if (_l1SkipSize > 0) { - uint8_t *l1Bytes = &l1SkipBytes[0]; - uint32_t l1BytesOffset = reinterpret_cast<unsigned long>(l1Bytes) & 7; - e.writeBits(reinterpret_cast<const uint64_t *>(l1Bytes - - l1BytesOffset), - l1BytesOffset * 8, - _l1SkipSize * 8); + writeZcBuf(e, l1SkipBytes); if (_l2SkipSize > 0) { - uint8_t *l2Bytes = &l2SkipBytes[0]; - uint32_t l2BytesOffset = - reinterpret_cast<unsigned long>(l2Bytes) & 7; - e.writeBits(reinterpret_cast<const uint64_t *>(l2Bytes - - l2BytesOffset), - l2BytesOffset * 8, - _l2SkipSize * 8); + writeZcBuf(e, l2SkipBytes); if (_l3SkipSize > 0) { - uint8_t *l3Bytes = &l3SkipBytes[0]; - uint32_t l3BytesOffset = - reinterpret_cast<unsigned long>(l3Bytes) & 7; - e.writeBits(reinterpret_cast<const uint64_t *>(l3Bytes - - l3BytesOffset), - l3BytesOffset * 8, - _l3SkipSize * 8); + writeZcBuf(e, l3SkipBytes); if (_l4SkipSize > 0) { - uint8_t *l4Bytes = &l4SkipBytes[0]; - uint32_t l4BytesOffset = - reinterpret_cast<unsigned long>(l4Bytes) & 7; - e.writeBits(reinterpret_cast<const uint64_t *>(l4Bytes - - l4BytesOffset), - l4BytesOffset * 8, - _l4SkipSize * 8); + writeZcBuf(e, l4SkipBytes); } } } @@ -493,20 +466,20 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, } -FakeZcFilterOcc::~FakeZcFilterOcc(void) +FakeZcFilterOcc::~FakeZcFilterOcc() { free(_compressedMalloc); } void -FakeZcFilterOcc::forceLink(void) +FakeZcFilterOcc::forceLink() { } size_t -FakeZcFilterOcc::bitSize(void) const +FakeZcFilterOcc::bitSize() const { return _compressedBits - (_l1SkipSize + _l2SkipSize + _l3SkipSize + _l4SkipSize) * 8; @@ -514,56 +487,56 @@ FakeZcFilterOcc::bitSize(void) const bool -FakeZcFilterOcc::hasWordPositions(void) const +FakeZcFilterOcc::hasWordPositions() const { return false; } size_t -FakeZcFilterOcc::skipBitSize(void) const +FakeZcFilterOcc::skipBitSize() const { return (_l1SkipSize + _l2SkipSize + _l3SkipSize + _l4SkipSize) * 8; } size_t -FakeZcFilterOcc::l1SkipBitSize(void) const +FakeZcFilterOcc::l1SkipBitSize() const { return _l1SkipSize * 8; } size_t -FakeZcFilterOcc::l2SkipBitSize(void) const +FakeZcFilterOcc::l2SkipBitSize() const { return _l2SkipSize * 8; } size_t -FakeZcFilterOcc::l3SkipBitSize(void) const +FakeZcFilterOcc::l3SkipBitSize() const { return _l3SkipSize * 8; } size_t -FakeZcFilterOcc::l4SkipBitSize(void) const +FakeZcFilterOcc::l4SkipBitSize() const { return _l4SkipSize * 8; } int -FakeZcFilterOcc::lowLevelSinglePostingScan(void) const +FakeZcFilterOcc::lowLevelSinglePostingScan() const { return 0; } int -FakeZcFilterOcc::lowLevelSinglePostingScanUnpack(void) const +FakeZcFilterOcc::lowLevelSinglePostingScanUnpack() const { return 0; } @@ -612,7 +585,7 @@ public: uint32_t docIdLimit, const fef::TermFieldMatchDataArray &matchData); - ~FakeFilterOccZCArrayIterator(void); + ~FakeFilterOccZCArrayIterator(); void doUnpack(uint32_t docId) override; void doSeek(uint32_t docId) override; @@ -702,7 +675,7 @@ FakeFilterOccZCArrayIterator::initRange(uint32_t begin, uint32_t end) FakeFilterOccZCArrayIterator:: -~FakeFilterOccZCArrayIterator(void) +~FakeFilterOccZCArrayIterator() { } @@ -789,7 +762,7 @@ FakeZcSkipFilterOcc<true>::FakeZcSkipFilterOcc(const FakeWord &fw) template <bool doSkip> -FakeZcSkipFilterOcc<doSkip>::~FakeZcSkipFilterOcc(void) +FakeZcSkipFilterOcc<doSkip>::~FakeZcSkipFilterOcc() { } @@ -842,7 +815,7 @@ public: uint32_t docIdLimit, const TermFieldMatchDataArray &matchData); - ~FakeFilterOccZCSkipArrayIterator(void); + ~FakeFilterOccZCSkipArrayIterator(); void doL4SkipSeek(uint32_t docId); void doL3SkipSeek(uint32_t docId); @@ -1021,7 +994,7 @@ initRange(uint32_t begin, uint32_t end) template <bool doSkip> FakeFilterOccZCSkipArrayIterator<doSkip>:: -~FakeFilterOccZCSkipArrayIterator(void) +~FakeFilterOccZCSkipArrayIterator() { } @@ -1390,7 +1363,7 @@ FakeEGCompr64PosOcc<bigEndian>::FakeEGCompr64PosOcc(const FakeWord &fw) template <bool bigEndian> -FakeEGCompr64PosOcc<bigEndian>::~FakeEGCompr64PosOcc(void) +FakeEGCompr64PosOcc<bigEndian>::~FakeEGCompr64PosOcc() { } @@ -1517,7 +1490,7 @@ FakeEG2Compr64PosOcc<bigEndian>::FakeEG2Compr64PosOcc(const FakeWord &fw) template <bool bigEndian> -FakeEG2Compr64PosOcc<bigEndian>::~FakeEG2Compr64PosOcc(void) +FakeEG2Compr64PosOcc<bigEndian>::~FakeEG2Compr64PosOcc() { } @@ -1596,7 +1569,7 @@ FakeEG2Compr64PosOcc<bigEndian>::setup(const FakeWord &fw) template <bool bigEndian> size_t -FakeEG2Compr64PosOcc<bigEndian>::bitSize(void) const +FakeEG2Compr64PosOcc<bigEndian>::bitSize() const { return _compressedBits; } @@ -1604,7 +1577,7 @@ FakeEG2Compr64PosOcc<bigEndian>::bitSize(void) const template <bool bigEndian> bool -FakeEG2Compr64PosOcc<bigEndian>::hasWordPositions(void) const +FakeEG2Compr64PosOcc<bigEndian>::hasWordPositions() const { return true; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp index 94ccd4cd891..6af2d61f110 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".fpfactory"); #include "fakeegcompr64filterocc.h" #include "fakefilterocc.h" #include "fakezcbfilterocc.h" @@ -11,11 +8,7 @@ LOG_SETUP(".fpfactory"); #include "fpfactory.h" #include "fakewordset.h" -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { using index::Schema; @@ -72,7 +65,7 @@ getFPFactory(const std::string &name, const Schema &schema) std::vector<std::string> -getPostingTypes(void) +getPostingTypes() { std::vector<std::string> res; @@ -93,7 +86,7 @@ FPFactoryInit::FPFactoryInit(const FPFactoryMapEntry &fpFactoryMapEntry) fpFactoryMap->insert(fpFactoryMapEntry); } -FPFactoryInit::~FPFactoryInit(void) +FPFactoryInit::~FPFactoryInit() { assert(fpFactoryMap != NULL); size_t eraseRes = fpFactoryMap->erase(_key); @@ -106,15 +99,13 @@ FPFactoryInit::~FPFactoryInit(void) } void -FPFactoryInit::forceLink(void) +FPFactoryInit::forceLink() { FakeEGCompr64FilterOcc::forceLink(); FakeFilterOcc::forceLink(); FakeZcbFilterOcc::forceLink(); FakeZcFilterOcc::forceLink(); FakeMemTreeOcc::forceLink(); -}; - -} // namespace fakedata +} -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.h b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.h index aa67eb1b933..e35a01269fa 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fpfactory.h @@ -19,7 +19,7 @@ class FPFactory { public: virtual - ~FPFactory(void); + ~FPFactory(); virtual FakePosting::SP make(const FakeWord &fw) = 0; @@ -62,7 +62,7 @@ FPFactory * getFPFactory(const std::string &name, const index::Schema &schema); std::vector<std::string> -getPostingTypes(void); +getPostingTypes(); class FPFactoryInit { @@ -70,10 +70,10 @@ class FPFactoryInit public: FPFactoryInit(const FPFactoryMapEntry &fpFactoryMapEntry); - ~FPFactoryInit(void); + ~FPFactoryInit(); static void - forceLink(void); + forceLink(); }; } // namespace fakedata diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h index 4b376df2bdd..eb278280156 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h @@ -15,6 +15,7 @@ #include <vespa/searchlib/query/queryterm.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/stringfmt.h> #include <algorithm> #include <future> #include <map> diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h index 5384df843f4..35463069c9f 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h @@ -99,7 +99,7 @@ public: } std::string - toStr(void) const + toStr() const { return _ss.str(); } diff --git a/searchlib/src/vespa/searchlib/test/statefile.cpp b/searchlib/src/vespa/searchlib/test/statefile.cpp index 005145c5cb3..9779cfa28a4 100644 --- a/searchlib/src/vespa/searchlib/test/statefile.cpp +++ b/searchlib/src/vespa/searchlib/test/statefile.cpp @@ -1,19 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "statefile.h" #include <vespa/searchlib/util/statefile.h> #include <iostream> #include <fstream> #include <string> -#include "statefile.h" - -namespace search -{ -namespace test -{ - -namespace statefile -{ +namespace search::test::statefile { vespalib::string readState(StateFile &sf) @@ -23,7 +15,6 @@ readState(StateFile &sf) return vespalib::string(buf.begin(), buf.end()); } - std::vector<vespalib::string> readHistory(const char *name) { @@ -40,9 +31,4 @@ readHistory(const char *name) return res; } - -} - -} - } diff --git a/searchlib/src/vespa/searchlib/test/statefile.h b/searchlib/src/vespa/searchlib/test/statefile.h index a5aee2ff2a3..6c67a112ebb 100644 --- a/searchlib/src/vespa/searchlib/test/statefile.h +++ b/searchlib/src/vespa/searchlib/test/statefile.h @@ -1,22 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/vespalib/stllike/string.h> +#include <vector> -namespace search -{ +namespace search { class StateFile; -namespace test -{ - -namespace statefile -{ +namespace test::statefile { vespalib::string readState(StateFile &sf); std::vector<vespalib::string> readHistory(const char *name); } - -} - } diff --git a/searchlib/src/vespa/searchlib/test/statestring.cpp b/searchlib/src/vespa/searchlib/test/statestring.cpp index e1c8df43c4b..0fb08415434 100644 --- a/searchlib/src/vespa/searchlib/test/statestring.cpp +++ b/searchlib/src/vespa/searchlib/test/statestring.cpp @@ -1,16 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <sstream> #include "statestring.h" +#include <sstream> -namespace search -{ - -namespace test -{ - -namespace statestring -{ +namespace search::test::statestring { bool testStartPos(vespalib::string &s, size_t pos) @@ -18,7 +10,6 @@ testStartPos(vespalib::string &s, size_t pos) return (pos < s.size() && (pos == 0 || s[pos - 1] == ' ')); } - size_t findStartPos(vespalib::string &s, const vespalib::string &key) { @@ -33,7 +24,6 @@ findStartPos(vespalib::string &s, const vespalib::string &key) return pos; } - size_t scanBreakPos(vespalib::string &s, size_t pos) { @@ -43,7 +33,6 @@ scanBreakPos(vespalib::string &s, size_t pos) return pos; } - void normalizeTimestamp(vespalib::string &s) { @@ -55,7 +44,6 @@ normalizeTimestamp(vespalib::string &s) } } - void normalizeAddr(vespalib::string &s, void *addr) { @@ -72,7 +60,6 @@ normalizeAddr(vespalib::string &s, void *addr) } } - void normalizeTimestamps(std::vector<vespalib::string> &sv) { @@ -81,7 +68,6 @@ normalizeTimestamps(std::vector<vespalib::string> &sv) } } - void normalizeAddrs(std::vector<vespalib::string> &sv, void *addr) { @@ -90,9 +76,4 @@ normalizeAddrs(std::vector<vespalib::string> &sv, void *addr) } } - -} - -} - } diff --git a/searchlib/src/vespa/searchlib/test/statestring.h b/searchlib/src/vespa/searchlib/test/statestring.h index ad5d70186dc..7fb6f1fce85 100644 --- a/searchlib/src/vespa/searchlib/test/statestring.h +++ b/searchlib/src/vespa/searchlib/test/statestring.h @@ -1,24 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/stllike/string.h> - -namespace search -{ -namespace test -{ +#include <vespa/vespalib/stllike/string.h> +#include <vector> -namespace statestring -{ +namespace search::test::statestring { void normalizeTimestamp(vespalib::string &s); void normalizeAddr(vespalib::string &s, void *addr); void normalizeTimestamps(std::vector<vespalib::string> &sv); void normalizeAddrs(std::vector<vespalib::string> &sv, void *addr); -} - -} - -} - - +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 9dad2216fcb..32eb5e5d252 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -390,7 +390,7 @@ int Domain::subscribe(const Domain::SP & domain, const SerialNum & from, FRT_Sup Domain::SerialNumList -Domain::scanDir(void) +Domain::scanDir() { SerialNumList res; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h index 1c081b8ecd7..88482cc1114 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h @@ -45,7 +45,7 @@ public: void sync(); SerialNumRange range() const { return _range; } - SerialNum getSynced(void) const { + SerialNum getSynced() const { vespalib::LockGuard guard(_writeLock); return _syncedSerial; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/nosyncproxy.cpp b/searchlib/src/vespa/searchlib/transactionlog/nosyncproxy.cpp index 2b8e2935752..5304697627b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/nosyncproxy.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/nosyncproxy.cpp @@ -1,20 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "nosyncproxy.h" -namespace search -{ -namespace transactionlog -{ +namespace search::transactionlog { -NoSyncProxy::NoSyncProxy(void) -{ -} +NoSyncProxy::NoSyncProxy() { } - -NoSyncProxy::~NoSyncProxy(void) -{ -} +NoSyncProxy::~NoSyncProxy() { } void @@ -24,5 +15,3 @@ NoSyncProxy::sync(SerialNum syncTo) } } - -} diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index fc5e94c1859..6e7e5e0d8f5 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/transactionlog/session.h> -#include <vespa/searchlib/transactionlog/domain.h> +#include "session.h" +#include "domain.h" +#include <vespa/fnet/frt/supervisor.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/log/log.h> @@ -96,6 +96,10 @@ Session::visitOnly() finalize(); } +bool Session::finished() const { + return _finished || (_connection->GetState() != FNET_Connection::FNET_CONNECTED); +} + void Session::enQ(const SP & session, SerialNum serial, const Packet & packet) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.h b/searchlib/src/vespa/searchlib/transactionlog/session.h index 46723bed04c..6c49d90d2fd 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.h +++ b/searchlib/src/vespa/searchlib/transactionlog/session.h @@ -2,11 +2,13 @@ #pragma once #include "common.h" -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/fnet/frt/invoker.h> #include <deque> +class FastOS_FileInterface; + namespace search { namespace transactionlog { @@ -30,7 +32,7 @@ public: bool inSync() const; bool continous() const { return _subscriber; } bool ok() const { return _ok; } - bool finished() const { return _finished || (_connection->GetState() != FNET_Connection::FNET_CONNECTED);} + bool finished() const; static void enQ(const SP & session, SerialNum serial, const Packet & packet); static Task::UP createTask(const Session::SP & session); private: diff --git a/searchlib/src/vespa/searchlib/transactionlog/syncproxy.h b/searchlib/src/vespa/searchlib/transactionlog/syncproxy.h index baf533518e7..4e52518b09e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/syncproxy.h +++ b/searchlib/src/vespa/searchlib/transactionlog/syncproxy.h @@ -12,13 +12,8 @@ namespace transactionlog class SyncProxy { public: - virtual - ~SyncProxy(void) - { - } - - virtual void - sync(SerialNum syncTo) = 0; + virtual ~SyncProxy() { } + virtual void sync(SerialNum syncTo) = 0; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp index fdc771a7b58..2fda267645f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "trans_log_server_explorer.h" #include "domain.h" #include <vespa/vespalib/data/slime/slime.h> diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp index df3aa1404ae..98b3e1b46ee 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp @@ -1,11 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "translogclient.h" -#include <stdexcept> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".translogclient"); +using namespace std::chrono_literals; + namespace search { namespace transactionlog { @@ -18,27 +23,31 @@ using vespalib::LockGuard; TransLogClient::TransLogClient(const vespalib::string & rpcTarget) : _rpcTarget(rpcTarget), _sessions(), - _supervisor(), + _supervisor(std::make_unique<FRT_Supervisor>()), _target(NULL) { reconnect(); - exportRPC(_supervisor); - _supervisor.Start(); + exportRPC(*_supervisor); + _supervisor->Start(); } TransLogClient::~TransLogClient() { disconnect(); - _supervisor.ShutDown(true); + _supervisor->ShutDown(true); } bool TransLogClient::reconnect() { disconnect(); - _target = _supervisor.Get2WayTarget(_rpcTarget.c_str()); + _target = _supervisor->Get2WayTarget(_rpcTarget.c_str()); return isConnected(); } +bool TransLogClient::isConnected() const { + return (_target != NULL) && _target->IsValid(); +} + void TransLogClient::disconnect() { if (_target) { @@ -48,7 +57,7 @@ void TransLogClient::disconnect() bool TransLogClient::create(const vespalib::string & domain) { - FRT_RPCRequest *req = _supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("createDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); @@ -58,7 +67,7 @@ bool TransLogClient::create(const vespalib::string & domain) bool TransLogClient::remove(const vespalib::string & domain) { - FRT_RPCRequest *req = _supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("deleteDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); @@ -69,7 +78,7 @@ bool TransLogClient::remove(const vespalib::string & domain) TransLogClient::Session::UP TransLogClient::open(const vespalib::string & domain) { Session::UP session; - FRT_RPCRequest *req = _supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("openDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); @@ -92,7 +101,7 @@ TransLogClient::Visitor::UP TransLogClient::createVisitor(const vespalib::string bool TransLogClient::listDomains(std::vector<vespalib::string> & dir) { - FRT_RPCRequest *req = _supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("listDomains"); int32_t retval(rpc(req)); if (retval == 0) { @@ -227,7 +236,7 @@ bool TransLogClient::Session::commit(const vespalib::ConstBufferRef & buf) { bool retval(true); if (buf.size() != 0) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainCommit"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddData(buf.c_str(), buf.size()); @@ -251,7 +260,7 @@ bool TransLogClient::Session::commit(const vespalib::ConstBufferRef & buf) bool TransLogClient::Session::status(SerialNum & b, SerialNum & e, size_t & count) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainStatus"); req->GetParams()->AddString(_domain.c_str()); int32_t retval(_tlc.rpc(req)); @@ -266,7 +275,7 @@ bool TransLogClient::Session::status(SerialNum & b, SerialNum & e, size_t & coun bool TransLogClient::Session::erase(const SerialNum & to) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainPrune"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt64(to); @@ -282,7 +291,7 @@ bool TransLogClient::Session::erase(const SerialNum & to) bool TransLogClient::Session::sync(const SerialNum &syncTo, SerialNum &syncedTo) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainSync"); FRT_Values & params = *req->GetParams(); params.AddString(_domain.c_str()); @@ -348,7 +357,7 @@ bool TransLogClient::Session::init(FRT_RPCRequest *req) bool TransLogClient::Visitor::visit(const SerialNum & from, const SerialNum & to) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainVisit"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt64(from); @@ -358,7 +367,7 @@ bool TransLogClient::Visitor::visit(const SerialNum & from, const SerialNum & to bool TransLogClient::Subscriber::subscribe(const SerialNum & from) { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainSubscribe"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt64(from); @@ -367,7 +376,7 @@ bool TransLogClient::Subscriber::subscribe(const SerialNum & from) bool TransLogClient::Session::run() { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainSessionRun"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt32(_sessionId); @@ -381,12 +390,12 @@ bool TransLogClient::Session::close() int retval(0); if (_sessionId > 0) { do { - FRT_RPCRequest *req = _tlc._supervisor.AllocRPCRequest(); + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); req->SetMethodName("domainSessionClose"); req->GetParams()->AddString(_domain.c_str()); req->GetParams()->AddInt32(_sessionId); if ( (retval = _tlc.rpc(req)) > 0) { - FastOS_Thread::Sleep(10); + std::this_thread::sleep_for(10ms); } req->SubRef(); } while ( retval == 1 ); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h index 802933b7d81..dbc53cc8608 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h @@ -3,14 +3,16 @@ #include "common.h" #include <vespa/document/util/bytebuffer.h> -#include <vespa/fnet/frt/frt.h> -#include <map> -#include <vector> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/buffer.h> +#include <vespa/fnet/frt/invokable.h> +#include <map> +#include <vector> + +class FRT_Supervisor; +class FRT_Target; -namespace search { -namespace transactionlog { +namespace search::transactionlog { class TransLogClient : private FRT_Invokable { @@ -102,7 +104,7 @@ public: Subscriber::UP createSubscriber(const vespalib::string & domain, Session::Callback & callBack); Visitor::UP createVisitor(const vespalib::string & domain, Session::Callback & callBack); - bool isConnected() const { return (_target != NULL) && _target->IsValid(); } + bool isConnected() const; void disconnect(); bool reconnect(); const vespalib::string &getRPCTarget() const { return _rpcTarget; } @@ -131,10 +133,9 @@ private: SessionMap _sessions; //Brute force lock for subscriptions. For multithread safety. vespalib::Lock _lock; - FRT_Supervisor _supervisor; + std::unique_ptr<FRT_Supervisor> _supervisor; FRT_Target * _target; }; } -} diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index cc904ef82a7..84f77c76f8c 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "translogserver.h" -#include <fstream> #include <vespa/vespalib/util/stringfmt.h> -#include <stdexcept> -#include <vespa/log/log.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fnet/frt/supervisor.h> +#include <fstream> +#include <vespa/log/log.h> LOG_SETUP(".transactionlog.server"); using vespalib::make_string; @@ -14,14 +14,9 @@ using vespalib::stringref; using vespalib::IllegalArgumentException; using search::common::FileHeaderContext; -namespace search -{ +namespace search::transactionlog { -namespace transactionlog -{ - -namespace -{ +namespace { class SyncHandler : public FNET_Task { @@ -36,8 +31,8 @@ public: const TransLogServer::Session::SP &session, SerialNum syncTo); - ~SyncHandler(void); - void PerformTask(void) override; + ~SyncHandler(); + void PerformTask() override; }; @@ -55,13 +50,13 @@ SyncHandler::SyncHandler(FRT_Supervisor *supervisor, } -SyncHandler::~SyncHandler(void) +SyncHandler::~SyncHandler() { } void -SyncHandler::PerformTask(void) +SyncHandler::PerformTask() { SerialNum synced(_domain->getSynced()); if (_session->getDown() || @@ -98,7 +93,7 @@ TransLogServer::TransLogServer(const vespalib::string &name, _defaultCrcType(defaultCrcType), _executor(maxThreads, 128*1024), _threadPool(8192, 1), - _supervisor(), + _supervisor(std::make_unique<FRT_Supervisor>()), _domains(), _reqQ(), _fileHeaderContext(fileHeaderContext) @@ -125,13 +120,13 @@ TransLogServer::TransLogServer(const vespalib::string &name, } } } - exportRPC(_supervisor); + exportRPC(*_supervisor); char listenSpec[32]; sprintf(listenSpec, "tcp/%d", listenPort); bool listenOk(false); for (int i(600); !listenOk && i; i--) { - if (_supervisor.Listen(listenSpec)) { - _supervisor.Start(); + if (_supervisor->Listen(listenSpec)) { + _supervisor->Start(); listenOk = true; } else { LOG(warning, "Failed listening at port %s trying for %d seconds more.", listenSpec, i); @@ -154,7 +149,7 @@ TransLogServer::~TransLogServer() { stop(); join(); - _supervisor.ShutDown(true); + _supervisor->ShutDown(true); } bool TransLogServer::onStop() @@ -256,9 +251,9 @@ TransLogServer::findDomain(const stringref &domainName) void TransLogServer::exportRPC(FRT_Supervisor & supervisor) { - _supervisor.SetSessionInitHook(FRT_METHOD(TransLogServer::initSession), this); - _supervisor.SetSessionFiniHook(FRT_METHOD(TransLogServer::finiSession), this); - _supervisor.SetSessionDownHook(FRT_METHOD(TransLogServer::downSession), this); + _supervisor->SetSessionInitHook(FRT_METHOD(TransLogServer::initSession), this); + _supervisor->SetSessionFiniHook(FRT_METHOD(TransLogServer::finiSession), this); + _supervisor->SetSessionDownHook(FRT_METHOD(TransLogServer::downSession), this); FRT_ReflectionBuilder rb( & supervisor); //-- Create Domain ----------------------------------------------------------- @@ -521,7 +516,7 @@ void TransLogServer::domainSubscribe(FRT_RPCRequest *req) if (domain) { SerialNum from(params[1]._intval64); LOG(debug, "domainSubscribe(%s, %" PRIu64 ")", domainName, from); - retval = domain->subscribe(domain, from, _supervisor, req->GetConnection()); + retval = domain->subscribe(domain, from, *_supervisor, req->GetConnection()); } ret.AddInt32(retval); } @@ -538,7 +533,7 @@ void TransLogServer::domainVisit(FRT_RPCRequest *req) SerialNum from(params[1]._intval64); SerialNum to(params[2]._intval64); LOG(debug, "domainVisit(%s, %" PRIu64 ", %" PRIu64 ")", domainName, from, to); - retval = domain->visit(domain, from, to, _supervisor, req->GetConnection()); + retval = domain->visit(domain, from, to, *_supervisor, req->GetConnection()); } ret.AddInt32(retval); } @@ -656,14 +651,9 @@ TransLogServer::domainSync(FRT_RPCRequest *req) return; } - SyncHandler *syncHandler = new SyncHandler(&_supervisor, - req, - domain, - session, - syncTo); + SyncHandler *syncHandler = new SyncHandler(_supervisor.get(), req, domain, session, syncTo); syncHandler->ScheduleNow(); } } -} diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index 105a6ca643b..1ef94bbee02 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -1,23 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchlib/transactionlog/domain.h> +#include "domain.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/document/util/queue.h> +#include <vespa/fnet/frt/invokable.h> #include <mutex> -namespace search -{ - -namespace common -{ +namespace search { -class FileHeaderContext; +namespace common { class FileHeaderContext; } -} - -namespace transactionlog -{ +namespace transactionlog { class TransLogServerExplorer; @@ -50,9 +44,9 @@ public: public: typedef std::shared_ptr<Session> SP; - Session(void) : _down(false) { } - bool getDown(void) const { return _down; } - void setDown(void) { _down = true; } + Session() : _down(false) { } + bool getDown() const { return _down; } + void setDown() { _down = true; } }; private: @@ -96,7 +90,7 @@ private: const DomainPart::Crc _defaultCrcType; vespalib::ThreadStackExecutor _executor; FastOS_ThreadPool _threadPool; - FRT_Supervisor _supervisor; + std::unique_ptr<FRT_Supervisor> _supervisor; DomainList _domains; mutable std::mutex _lock; // Protects _domains std::mutex _fileLock; // Protects the creating and deleting domains including file system operations. diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp index fdfd9189cd8..1b0959d63fd 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserverapp.cpp @@ -1,17 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/transactionlog/translogserverapp.h> + +#include "translogserverapp.h" #include <vespa/log/log.h> LOG_SETUP(".translogserverapp"); using search::common::FileHeaderContext; -namespace search -{ - -namespace transactionlog -{ +namespace search { +namespace transactionlog { TransLogServerApp::TransLogServerApp(const config::ConfigUri & tlsConfigUri, const FileHeaderContext & fileHeaderContext) diff --git a/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp b/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp index 59e990f6df0..88cd4fb5817 100644 --- a/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/uca/ucafunctionnode.h> -#include <vespa/searchlib/uca/ucaconverter.h> +#include "ucafunctionnode.h" +#include "ucaconverter.h" namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.cpp b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp index 21e1d16a079..af27001aba3 100644 --- a/searchlib/src/vespa/searchlib/util/bufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "bufferwriter.h" -namespace search -{ +namespace search { BufferWriter::BufferWriter() : _cur(nullptr), diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.h b/searchlib/src/vespa/searchlib/util/bufferwriter.h index b4183f779ed..f0a0557da67 100644 --- a/searchlib/src/vespa/searchlib/util/bufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.h @@ -2,8 +2,9 @@ #pragma once -namespace search -{ +#include <cstddef> + +namespace search { /** * Abstract class to write to a buffer with an abstract backing store diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index 5d6927834b9..19703f58d3b 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -11,7 +11,7 @@ namespace search { class ComprBuffer { private: - void allocComprBuf(void); + void allocComprBuf(); public: void *_comprBuf; size_t _comprBufSize; diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 719b423861c..d0b78e6bdd0 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -2,16 +2,12 @@ // Copyright (C) 2002-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/searchlib/util/comprbuffer.h> -#include <vespa/searchlib/util/comprfile.h> +#include "comprfile.h" #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/fastos/file.h> +#include <cassert> -LOG_SETUP(".comprbuffer"); - -namespace search -{ +namespace search { using vespalib::nbostream; @@ -321,7 +317,7 @@ ComprFileReadContext(uint32_t unitSize) } -ComprFileReadContext::~ComprFileReadContext(void) +ComprFileReadContext::~ComprFileReadContext() { } @@ -341,7 +337,7 @@ ComprFileReadContext::readComprBuffer(uint64_t stopOffset, bool readAll) void -ComprFileReadContext::readComprBuffer(void) +ComprFileReadContext::readComprBuffer() { search::ComprFileReadBase::ReadComprBuffer(_stopOffset, _readAll, @@ -545,7 +541,7 @@ ComprFileWriteContext(uint32_t unitSize) } -ComprFileWriteContext::~ComprFileWriteContext(void) +ComprFileWriteContext::~ComprFileWriteContext() { } @@ -599,7 +595,7 @@ ComprFileWriteContext::allocComprBuf(unsigned int comprBufSize, void -ComprFileWriteContext::allocComprBuf(void) +ComprFileWriteContext::allocComprBuf() { allocComprBuf(32768, 32768); } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index a30583e2c1b..63db1bfe7f5 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -51,7 +51,7 @@ public: */ virtual void setupBits(int bitOffset) = 0; virtual uint64_t getBitPos(int bitOffset, uint64_t bufferEndFilePos) const = 0; - virtual uint64_t getBitPosV(void) const = 0; + virtual uint64_t getBitPosV() const = 0; virtual void skipBits(int bits) = 0; virtual void adjUnitPtr(int newRemainingUnits) = 0; virtual void emptyBuffer(uint64_t newBitPosition) = 0; @@ -59,7 +59,7 @@ public: /** * Get size of each unit (typically 4 or 8) */ - virtual uint32_t getUnitByteSize(void) const = 0; + virtual uint32_t getUnitByteSize() const = 0; /** * Checkpoint write. Used at semi-regular intervals during indexing @@ -97,7 +97,7 @@ public: ComprBuffer &cbuf); protected: - virtual ~ComprFileReadBase(void) { } + virtual ~ComprFileReadBase() { } }; @@ -229,7 +229,7 @@ public: */ virtual void checkPointRead(vespalib::nbostream &in) = 0; - virtual uint64_t getBitPosV(void) const = 0; + virtual uint64_t getBitPosV() const = 0; }; class ComprFileWriteBase @@ -260,16 +260,16 @@ public: void writeComprBuffer(bool flushSlack); void allocComprBuf(unsigned int comprBufSize, size_t preferredFileAlignment); - void allocComprBuf(void); + void allocComprBuf(); void setEncodeContext(ComprFileEncodeContext *encodeContext) { _encodeContext = encodeContext; } - ComprFileEncodeContext *getEncodeContext(void) const { return _encodeContext; } + ComprFileEncodeContext *getEncodeContext() const { return _encodeContext; } void setFile(FastOS_FileInterface *file) { _file = file; } - FastOS_FileInterface *getFile(void) const { return _file; } + FastOS_FileInterface *getFile() const { return _file; } /** * Get file offset for start of compressed buffer. */ - uint64_t getBufferStartFilePos(void) const { return _fileWriteByteOffset; } + uint64_t getBufferStartFilePos() const { return _fileWriteByteOffset; } /** * Set file offset for start of compressed byffer. diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp index a6c716a13cd..c5260c7f454 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp @@ -2,11 +2,11 @@ // Copyright (C) 2002-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include "dirtraverse.h" +#include <vespa/fastos/file.h> +#include <cassert> -namespace search -{ +namespace search { extern "C" { static int cmpname(const void *av, const void *bv) @@ -19,10 +19,16 @@ static int cmpname(const void *av, const void *bv) } } +DirectoryTraverse::Name::Name(const char *name) + : _name(nullptr), + _next(nullptr) +{ + _name = strdup(name); +} +DirectoryTraverse::Name::~Name() { free(_name); } DirectoryTraverse::Name * -DirectoryTraverse::Name::sort(Name *head, - int count) +DirectoryTraverse::Name::sort(Name *head, int count) { Name *nl; Name **names; @@ -30,7 +36,7 @@ DirectoryTraverse::Name::sort(Name *head, names = new Name *[count]; i = 0; - for(nl = head; nl != NULL; nl = nl->_next) + for(nl = head; nl != nullptr; nl = nl->_next) names[i++] = nl; assert(i == count); qsort(names, count, sizeof(Name *), cmpname); @@ -38,7 +44,7 @@ DirectoryTraverse::Name::sort(Name *head, if (i + 1 < count) names[i]->_next = names[i + 1]; else - names[i]->_next = NULL; + names[i]->_next = nullptr; } head = names[0]; delete [] names; @@ -50,7 +56,7 @@ void DirectoryTraverse::QueueDir(const char *name) { Name *n = new Name(name); - if (_dirTail == NULL) + if (_dirTail == nullptr) _dirHead = n; else _dirTail->_next = n; @@ -77,59 +83,59 @@ DirectoryTraverse::PushRemoveDir(const char *name) void -DirectoryTraverse::PushPushedDirs(void) +DirectoryTraverse::PushPushedDirs() { Name *n; - while (_pdirHead != NULL) { + while (_pdirHead != nullptr) { n = _pdirHead; _pdirHead = n->_next; n->_next = _dirHead; _dirHead = n; - if (_dirTail == NULL) + if (_dirTail == nullptr) _dirTail = n; } } DirectoryTraverse::Name * -DirectoryTraverse::UnQueueDir(void) +DirectoryTraverse::UnQueueDir() { Name *n; PushPushedDirs(); - if (_dirHead == NULL) - return NULL; + if (_dirHead == nullptr) + return nullptr; n = _dirHead; _dirHead = n->_next; - n->_next = NULL; - if (_dirHead == NULL) - _dirTail = NULL; + n->_next = nullptr; + if (_dirHead == nullptr) + _dirTail = nullptr; return n; } DirectoryTraverse::Name * -DirectoryTraverse::UnQueueName(void) +DirectoryTraverse::UnQueueName() { Name *n; - if (_nameHead == NULL) - return NULL; + if (_nameHead == nullptr) + return nullptr; n = _nameHead; _nameHead = n->_next; - n->_next = NULL; + n->_next = nullptr; _nameCount--; return n; } void -DirectoryTraverse::ScanSingleDir(void) +DirectoryTraverse::ScanSingleDir() { - assert(_nameHead == NULL); + assert(_nameHead == nullptr); assert(_nameCount == 0); delete _curDir; free(_fullDirName); - _fullDirName = NULL; + _fullDirName = nullptr; _curDir = UnQueueDir(); - if (_curDir == NULL) + if (_curDir == nullptr) return; _fullDirName = (char *) malloc(strlen(_baseDir) + 1 + strlen(_curDir->_name) + 1); @@ -156,13 +162,13 @@ DirectoryTraverse::ScanSingleDir(void) bool -DirectoryTraverse::NextName(void) +DirectoryTraverse::NextName() { delete _curName; - _curName = NULL; - while (_nameHead == NULL && (_dirHead != NULL || _pdirHead != NULL)) + _curName = nullptr; + while (_nameHead == nullptr && (_dirHead != nullptr || _pdirHead != nullptr)) ScanSingleDir(); - if (_nameHead == NULL) + if (_nameHead == nullptr) return false; _curName = UnQueueName(); free(_fullName); @@ -177,13 +183,13 @@ DirectoryTraverse::NextName(void) bool -DirectoryTraverse::NextRemoveDir(void) +DirectoryTraverse::NextRemoveDir() { Name *curName; delete _curName; - _curName = NULL; - if (_rdirHead == NULL) + _curName = nullptr; + if (_rdirHead == nullptr) return false; curName = _rdirHead; _rdirHead = curName->_next; @@ -200,7 +206,7 @@ DirectoryTraverse::NextRemoveDir(void) bool -DirectoryTraverse::RemoveTree(void) +DirectoryTraverse::RemoveTree() { FastOS_StatInfo statInfo; @@ -246,18 +252,18 @@ DirectoryTraverse::GetTreeSize() } DirectoryTraverse::DirectoryTraverse(const char *baseDir) - : _baseDir(NULL), - _nameHead(NULL), + : _baseDir(nullptr), + _nameHead(nullptr), _nameCount(0), - _dirHead(NULL), - _dirTail(NULL), - _pdirHead(NULL), - _rdirHead(NULL), - _curDir(NULL), - _curName(NULL), - _fullDirName(NULL), - _fullName(NULL), - _relName(NULL) + _dirHead(nullptr), + _dirTail(nullptr), + _pdirHead(nullptr), + _rdirHead(nullptr), + _curDir(nullptr), + _curName(nullptr), + _fullDirName(nullptr), + _fullName(nullptr), + _relName(nullptr) { _baseDir = strdup(baseDir); QueueDir(""); @@ -265,7 +271,7 @@ DirectoryTraverse::DirectoryTraverse(const char *baseDir) } -DirectoryTraverse::~DirectoryTraverse(void) +DirectoryTraverse::~DirectoryTraverse() { free(_fullDirName); free(_fullName); @@ -273,15 +279,15 @@ DirectoryTraverse::~DirectoryTraverse(void) delete _curDir; delete _curName; PushPushedDirs(); - while (_dirHead != NULL) + while (_dirHead != nullptr) delete UnQueueDir(); - while (_nameHead != NULL) + while (_nameHead != nullptr) delete UnQueueName(); - while (_rdirHead != NULL) { + while (_rdirHead != nullptr) { Name *n; n = _rdirHead; _rdirHead = n->_next; - n->_next = NULL; + n->_next = nullptr; delete n; } } diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.h b/searchlib/src/vespa/searchlib/util/dirtraverse.h index 550da2fa7d1..6351e20230f 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.h +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.h @@ -4,8 +4,9 @@ #pragma once -namespace search -{ +#include <cstdint> + +namespace search { class DirectoryTraverse { @@ -23,13 +24,8 @@ public: public: char *_name; Name *_next; - explicit Name(const char *name) - : _name(NULL), - _next(NULL) - { - _name = strdup(name); - } - ~Name(void) { free(_name); } + explicit Name(const char *name); + ~Name(); static Name *sort(Name *head, int count); }; private: @@ -46,22 +42,21 @@ private: char *_fullName; char *_relName; public: - const char *GetFullName(void) const { return _fullName; } - const char *GetRelName(void) const { return _relName; } + const char *GetFullName() const { return _fullName; } + const char *GetRelName() const { return _relName; } void QueueDir(const char *name); void PushDir(const char *name); void PushRemoveDir(const char *name); - void PushPushedDirs(void); - Name *UnQueueDir(void); - Name *UnQueueName(void); - void ScanSingleDir(void); - bool NextName(void); - bool NextRemoveDir(void); - bool RemoveTree(void); + void PushPushedDirs(); + Name *UnQueueDir(); + Name *UnQueueName(); + void ScanSingleDir(); + bool NextName(); + bool NextRemoveDir(); + bool RemoveTree(); uint64_t GetTreeSize(); // Returns size of directory in bytes explicit DirectoryTraverse(const char *baseDir); - ~DirectoryTraverse(void); + ~DirectoryTraverse(); }; } // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.cpp b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.cpp index 9c1150917a7..db721f5e9b6 100644 --- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "drainingbufferwriter.h" +#include <cassert> -namespace search -{ +namespace search { DrainingBufferWriter::DrainingBufferWriter() : BufferWriter(), diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h index 2c471ce2148..e02c30c7ca8 100644 --- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h @@ -4,9 +4,9 @@ #include "bufferwriter.h" #include <vector> +#include <cstdint> -namespace search -{ +namespace search { /** * Class to write to a "drain" buffer, used to measure performance of @@ -21,11 +21,8 @@ public: static constexpr size_t BUFFER_SIZE = 262144; DrainingBufferWriter(); - - virtual ~DrainingBufferWriter(); - - virtual void flush() override; - + ~DrainingBufferWriter() override; + void flush() override; size_t getBytesWritten() const { return _bytesWritten; } }; diff --git a/searchlib/src/vespa/searchlib/util/fileheadertk.cpp b/searchlib/src/vespa/searchlib/util/fileheadertk.cpp index 6451df32b69..fbaf5712a29 100644 --- a/searchlib/src/vespa/searchlib/util/fileheadertk.cpp +++ b/searchlib/src/vespa/searchlib/util/fileheadertk.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/component/vtag.h> + #include "fileheadertk.h" +#include <vespa/vespalib/component/vtag.h> using namespace search; using vespalib::GenericHeader; diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp index df509f2ea96..c5b35f92f0c 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.cpp +++ b/searchlib/src/vespa/searchlib/util/filekit.cpp @@ -2,16 +2,14 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/util/error.h> -#include <memory> -#include <string> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> LOG_SETUP(".filekit"); -namespace search -{ +namespace search { using vespalib::getLastErrorString; diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h index fb2332529d4..77dbe92a200 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.h +++ b/searchlib/src/vespa/searchlib/util/filekit.h @@ -5,6 +5,7 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <vespa/fastos/timestamp.h> namespace search { @@ -13,23 +14,15 @@ class FileKit private: static bool _syncFiles; public: - static bool - createStamp(const vespalib::stringref &name); - - static bool - hasStamp(const vespalib::stringref &name); - - static bool - removeStamp(const vespalib::stringref &name); + static bool createStamp(const vespalib::stringref &name); + static bool hasStamp(const vespalib::stringref &name); + static bool removeStamp(const vespalib::stringref &name); /** * Returns the modification time of the given file/directory, * or time stamp 0 if stating of file/directory fails. **/ - static fastos::TimeStamp - getModificationTime(const vespalib::stringref &name); - + static fastos::TimeStamp getModificationTime(const vespalib::stringref &name); }; } - diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp index 4ae53c03430..4be1eb668c9 100644 --- a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp +++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp @@ -1,14 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "filesizecalculator.h" +#include <vespa/vespalib/data/fileheader.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.filesizecalculator"); -#include "filesizecalculator.h" -#include <vespa/vespalib/data/fileheader.h> -namespace search -{ +namespace search { namespace { @@ -33,22 +31,19 @@ FileSizeCalculator::extractFileSize(const vespalib::GenericHeader &header, uint64_t fileByteSize = fileBitSize / 8; if (!byteAligned(fileBitSize)) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " which is not a multiple of 8", + "Bad header file size tag for %s, fileBitSize=%zu which is not a multiple of 8", fileName.c_str(), fileBitSize); return false; } if (fileByteSize < headerLen) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " but header is %" PRIu64 "bits", + "Bad header file size tag for %s, fileBitSize=%zu but header is %zu bits", fileName.c_str(), fileBitSize, headerLen * 8); return false; } if (fileByteSize > fileSize) { LOG(error, - "Bad header file size tag for %s, fileBitSize=%" PRIu64 - " but whole file size is %" PRIu64 "bits", + "Bad header file size tag for %s, fileBitSize=%zu but whole file size is %zu bits", fileName.c_str(), fileBitSize, fileSize * 8); return false; } diff --git a/searchlib/src/vespa/searchlib/util/foldedstringcompare.cpp b/searchlib/src/vespa/searchlib/util/foldedstringcompare.cpp index ac63d1a7a64..068e58e0591 100644 --- a/searchlib/src/vespa/searchlib/util/foldedstringcompare.cpp +++ b/searchlib/src/vespa/searchlib/util/foldedstringcompare.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "foldedstringcompare.h" #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> diff --git a/searchlib/src/vespa/searchlib/util/foldedstringcompare.h b/searchlib/src/vespa/searchlib/util/foldedstringcompare.h index 1a4945335c0..568305255e7 100644 --- a/searchlib/src/vespa/searchlib/util/foldedstringcompare.h +++ b/searchlib/src/vespa/searchlib/util/foldedstringcompare.h @@ -2,14 +2,14 @@ #pragma once - +#include <cstddef> namespace search { class FoldedStringCompare { public: - FoldedStringCompare(void) {} + FoldedStringCompare() {} /** * count number of UCS-4 characters in utf8 string @@ -54,4 +54,3 @@ public: }; } // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/ioerrorhandler.cpp b/searchlib/src/vespa/searchlib/util/ioerrorhandler.cpp index a7f548a7c67..23fa3529599 100644 --- a/searchlib/src/vespa/searchlib/util/ioerrorhandler.cpp +++ b/searchlib/src/vespa/searchlib/util/ioerrorhandler.cpp @@ -1,26 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "ioerrorhandler.h" #include "statebuf.h" #include "statefile.h" -#include <atomic> - -namespace search -{ +#include <vespa/fastos/file.h> +namespace search { IOErrorHandler *IOErrorHandler::_instance = nullptr; -namespace -{ +namespace { std::atomic<int> nesting; } void -IOErrorHandler::trap(void) +IOErrorHandler::trap() { _instance = this; FastOS_File::SetFailedHandler(forward); @@ -29,7 +25,7 @@ IOErrorHandler::trap(void) void -IOErrorHandler::untrap(void) +IOErrorHandler::untrap() { #ifdef notyet FastOS_File::SetFailedHandler(nullptr); diff --git a/searchlib/src/vespa/searchlib/util/ioerrorhandler.h b/searchlib/src/vespa/searchlib/util/ioerrorhandler.h index acdc0d12282..1feb0045c4f 100644 --- a/searchlib/src/vespa/searchlib/util/ioerrorhandler.h +++ b/searchlib/src/vespa/searchlib/util/ioerrorhandler.h @@ -2,8 +2,11 @@ #pragma once -namespace search -{ +#include <cstdint> +#include <cstddef> +#include <sys/types.h> + +namespace search { class StateFile; @@ -17,44 +20,22 @@ class IOErrorHandler bool _trapped; bool _fired; - using FailedHandler = void (*)(const char *op, - const char *file, - int error, - int64_t offset, - size_t len, - ssize_t rlen); - void - trap(); - - void - untrap(); - - static void - forward(const char *op, - const char *file, - int error, - int64_t offset, - size_t len, - ssize_t rlen); - - void - handle(const char *op, - const char *file, - int error, - int64_t offset, - size_t len, - ssize_t rlen); + using FailedHandler = void (*)(const char *op, const char *file, int error, + int64_t offset, size_t len, ssize_t rlen); + void trap(); + void untrap(); + + static void forward(const char *op, const char *file, int error, + int64_t offset, size_t len, ssize_t rlen); + + void handle(const char *op, const char *file, int error, + int64_t offset, size_t len, ssize_t rlen); public: IOErrorHandler(StateFile *stateFile); - ~IOErrorHandler(); - bool - fired() const - { - return _fired; - } + bool fired() const { return _fired; } }; diff --git a/searchlib/src/vespa/searchlib/util/logutil.cpp b/searchlib/src/vespa/searchlib/util/logutil.cpp index f8dd7120aac..db51fe8a155 100644 --- a/searchlib/src/vespa/searchlib/util/logutil.cpp +++ b/searchlib/src/vespa/searchlib/util/logutil.cpp @@ -1,7 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.util.logutil"); #include "logutil.h" #include <vespa/searchlib/util/dirtraverse.h> diff --git a/searchlib/src/vespa/searchlib/util/memoryusage.h b/searchlib/src/vespa/searchlib/util/memoryusage.h index 1cbb1fb0fd1..bf179672f59 100644 --- a/searchlib/src/vespa/searchlib/util/memoryusage.h +++ b/searchlib/src/vespa/searchlib/util/memoryusage.h @@ -28,10 +28,10 @@ public: _allocatedBytesOnHold(onHold) { } - size_t allocatedBytes(void) const { return _allocatedBytes; } - size_t usedBytes(void) const { return _usedBytes; } - size_t deadBytes(void) const { return _deadBytes; } - size_t allocatedBytesOnHold(void) const { return _allocatedBytesOnHold; } + size_t allocatedBytes() const { return _allocatedBytes; } + size_t usedBytes() const { return _usedBytes; } + size_t deadBytes() const { return _deadBytes; } + size_t allocatedBytesOnHold() const { return _allocatedBytesOnHold; } void incAllocatedBytes(size_t inc) { _allocatedBytes += inc; } void decAllocatedBytes(size_t dec) { _allocatedBytes -= dec; } void incUsedBytes(size_t inc) { _usedBytes += inc; } diff --git a/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h b/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h index 42519e10b9f..4f444d93cb9 100644 --- a/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h +++ b/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h @@ -30,7 +30,7 @@ public: } IN * - get(void) const + get() const { return _ref; } @@ -39,19 +39,19 @@ public: typedef std::vector<Ref> Vector; Vector _vec; - PostingPriorityQueue(void) + PostingPriorityQueue() : _vec() { } bool - empty(void) const + empty() const { return _vec.empty(); } void - clear(void) + clear() { _vec.clear(); } @@ -67,7 +67,7 @@ public: * and adjust() can be used. */ void - sort(void) + sort() { std::sort(_vec.begin(), _vec.end()); } @@ -76,7 +76,7 @@ public: * Return lowest value. Assumes vector is sorted. */ IN * - lowest(void) const + lowest() const { return _vec.front().get(); } @@ -86,7 +86,7 @@ public: * value. Perform adjustments to make vector sorted again. */ void - adjust(void); + adjust(); template <class OUT> @@ -116,7 +116,7 @@ public: template <class IN> void -PostingPriorityQueue<IN>::adjust(void) +PostingPriorityQueue<IN>::adjust() { typedef typename Vector::iterator VIT; if (!_vec.front().get()->isValid()) { diff --git a/searchlib/src/vespa/searchlib/util/rand48.h b/searchlib/src/vespa/searchlib/util/rand48.h index 6d27d179a74..9f7118aca27 100644 --- a/searchlib/src/vespa/searchlib/util/rand48.h +++ b/searchlib/src/vespa/searchlib/util/rand48.h @@ -21,7 +21,7 @@ public: _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; } - Rand48(void) + Rand48() : _state(0) { srand48(0x1234abcd); diff --git a/searchlib/src/vespa/searchlib/util/randomgenerator.h b/searchlib/src/vespa/searchlib/util/randomgenerator.h index b5cf60a4899..002d215b92a 100644 --- a/searchlib/src/vespa/searchlib/util/randomgenerator.h +++ b/searchlib/src/vespa/searchlib/util/randomgenerator.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vector> +#include <cassert> namespace search { class RandomGenerator diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.cpp b/searchlib/src/vespa/searchlib/util/rawbuf.cpp index 84a20ecf258..4954c9b3955 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.cpp +++ b/searchlib/src/vespa/searchlib/util/rawbuf.cpp @@ -2,21 +2,21 @@ // Copyright (C) 1998-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS - -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/util/rawbuf.h> +#include "rawbuf.h" #include <vespa/vespalib/util/compress.h> +#include <vespa/fastos/file.h> +#include <cassert> namespace search { static inline size_t smin(size_t a, size_t b) { return (a < b) ? a : b; } RawBuf::RawBuf(size_t size) - : _bufStart(NULL), - _bufEnd(NULL), - _bufFillPos(NULL), - _bufDrainPos(NULL), - _initialBufStart(NULL), + : _bufStart(nullptr), + _bufEnd(nullptr), + _bufFillPos(nullptr), + _bufDrainPos(nullptr), + _initialBufStart(nullptr), _initialSize(size) { if (size > 0) { @@ -28,10 +28,10 @@ RawBuf::RawBuf(size_t size) RawBuf::RawBuf(char *start, size_t size) - : _bufStart(NULL), - _bufEnd(NULL), - _bufFillPos(NULL), - _bufDrainPos(NULL), + : _bufStart(nullptr), + _bufEnd(nullptr), + _bufFillPos(nullptr), + _bufDrainPos(nullptr), _initialBufStart(start), _initialSize(size) { @@ -41,7 +41,7 @@ RawBuf::RawBuf(char *start, size_t size) } -RawBuf::~RawBuf(void) +RawBuf::~RawBuf() { if (_bufStart != _initialBufStart) free(_bufStart); @@ -113,7 +113,7 @@ RawBuf::appendCompressedNumber(int64_t n) * Has the entire contents of the buffer been used up, i.e. freed? */ bool -RawBuf::IsEmpty(void) +RawBuf::IsEmpty() { return _bufFillPos == _bufDrainPos; } @@ -159,7 +159,7 @@ RawBuf::preAlloc(size_t len) void -RawBuf::Compact(void) +RawBuf::Compact() { if (_bufDrainPos == _bufStart) return; @@ -171,18 +171,18 @@ RawBuf::Compact(void) void -RawBuf::Reuse(void) +RawBuf::Reuse() { if (static_cast<size_t>(_bufEnd - _bufStart) > _initialSize * 4) { free(_bufStart); if (_initialSize > 0) { - if (_initialBufStart != NULL) + if (_initialBufStart != nullptr) _bufStart = _initialBufStart; else _bufStart = static_cast<char *>(malloc(_initialSize)); - assert(_bufStart != NULL); + assert(_bufStart != nullptr); } else - _bufStart = NULL; + _bufStart = nullptr; _bufEnd = _bufStart + _initialSize; } _bufDrainPos = _bufFillPos = _bufStart; @@ -357,4 +357,10 @@ RawBuf::readFile(FastOS_FileInterface &file, size_t maxlen) return got; } +void +RawBuf::ensureSizeInternal(size_t size) { + expandBuf(size); + assert(static_cast<size_t>(_bufEnd - _bufFillPos) >= size); +} + } diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.h b/searchlib/src/vespa/searchlib/util/rawbuf.h index 5639914bc80..03378665189 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.h +++ b/searchlib/src/vespa/searchlib/util/rawbuf.h @@ -5,8 +5,7 @@ #pragma once #include <vespa/searchlib/common/hitrank.h> -#include <stdint.h> -#include <assert.h> +#include <cstdint> #include <sys/types.h> class FastOS_FileInterface; @@ -31,11 +30,13 @@ private: char* _bufDrainPos; // ref. to next byte to take out of buffer char* _initialBufStart; size_t _initialSize; + + void ensureSizeInternal(size_t size); public: RawBuf(char *start, size_t size);// Initially use provided buffer RawBuf(size_t size); // malloc-s given size, assigns to _bufStart - ~RawBuf(void); // Frees _bufStart, i.e. the char[]. + ~RawBuf(); // Frees _bufStart, i.e. the char[]. void operator+=(const char *src); void operator+=(const RawBuf& buffer); @@ -52,30 +53,29 @@ public: void appendLong(uint64_t n); void appendCompressedPositiveNumber(uint64_t n); void appendCompressedNumber(int64_t n); - bool IsEmpty(void); // Return whether all written. + bool IsEmpty(); // Return whether all written. void expandBuf(size_t needlen); - size_t GetFreeLen(void) const { return _bufEnd - _bufFillPos; } - size_t GetDrainLen(void) const { return _bufDrainPos - _bufStart; } - const char *GetDrainPos(void) const { return _bufDrainPos; } - const char *GetFillPos(void) const { return _bufFillPos; } - char * GetWritableFillPos(void) const { return _bufFillPos; } + size_t GetFreeLen() const { return _bufEnd - _bufFillPos; } + size_t GetDrainLen() const { return _bufDrainPos - _bufStart; } + const char *GetDrainPos() const { return _bufDrainPos; } + const char *GetFillPos() const { return _bufFillPos; } + char * GetWritableFillPos() const { return _bufFillPos; } char * GetWritableFillPos(size_t len) { preAlloc(len); return _bufFillPos; } char * GetWritableDrainPos(size_t offset) { return _bufDrainPos + offset; } void truncate(size_t offset) { _bufFillPos = _bufDrainPos + offset; } void preAlloc(size_t len); // Ensure room for 'len' more bytes. size_t readFile(FastOS_FileInterface &file, size_t maxlen); - void reset(void) { _bufDrainPos = _bufFillPos = _bufStart; } - void Compact(void); - void Reuse(void); - size_t GetUsedAndDrainLen(void) const { return _bufFillPos - _bufStart; } - size_t GetUsedLen(void) const { return _bufFillPos - _bufDrainPos; } + void reset() { _bufDrainPos = _bufFillPos = _bufStart; } + void Compact(); + void Reuse(); + size_t GetUsedAndDrainLen() const { return _bufFillPos - _bufStart; } + size_t GetUsedLen() const { return _bufFillPos - _bufDrainPos; } void Drain(size_t len); // Adjust drain pos. void Fill(size_t len) { _bufFillPos += len; } void ensureSize(size_t size) { if (static_cast<size_t>(_bufEnd - _bufFillPos) < size) { - expandBuf(size); - assert(static_cast<size_t>(_bufEnd - _bufFillPos) >= size); + ensureSizeInternal(size); } } @@ -144,4 +144,3 @@ public: }; } - diff --git a/searchlib/src/vespa/searchlib/util/sigbushandler.cpp b/searchlib/src/vespa/searchlib/util/sigbushandler.cpp index 7670388572a..fa7958a853b 100644 --- a/searchlib/src/vespa/searchlib/util/sigbushandler.cpp +++ b/searchlib/src/vespa/searchlib/util/sigbushandler.cpp @@ -1,23 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include "sigbushandler.h" #include "statefile.h" #include "statebuf.h" -#include "sigbushandler.h" -#include <setjmp.h> -#include <system_error> -#include <mutex> -#include <atomic> - -namespace search -{ +#include <unistd.h> +#include <cstring> +namespace search { SigBusHandler *SigBusHandler::_instance = nullptr; - -namespace -{ +namespace { std::atomic<int> sigBusNesting; @@ -32,18 +25,13 @@ public: _gotLock = sigBusNesting.compare_exchange_strong(expzero, 1); } - ~TryLockGuard() noexcept - { + ~TryLockGuard() noexcept { if (_gotLock) { sigBusNesting = 0; } } - bool - gotLock() const noexcept - { - return _gotLock; - } + bool gotLock() const noexcept { return _gotLock; } }; @@ -63,7 +51,7 @@ mystderr(const char *msg) noexcept } void -SigBusHandler::trap(void) +SigBusHandler::trap() { struct sigaction sa; _instance = this; @@ -78,7 +66,7 @@ SigBusHandler::trap(void) void -SigBusHandler::untrap(void) +SigBusHandler::untrap() { struct sigaction sa; memset(&sa, 0, sizeof(sa)); @@ -158,11 +146,9 @@ SigBusHandler::SigBusHandler(StateFile *stateFile) trap(); } - SigBusHandler::~SigBusHandler() { untrap(); } - } diff --git a/searchlib/src/vespa/searchlib/util/sigbushandler.h b/searchlib/src/vespa/searchlib/util/sigbushandler.h index 49c7879b10a..b41cc0d606b 100644 --- a/searchlib/src/vespa/searchlib/util/sigbushandler.h +++ b/searchlib/src/vespa/searchlib/util/sigbushandler.h @@ -2,10 +2,10 @@ #pragma once -#include <setjmp.h> +#include <csetjmp> +#include <csignal> -namespace search -{ +namespace search { class StateFile; @@ -22,39 +22,20 @@ class SigBusHandler bool _fired; char _buf[2048]; - void - trap(); - - void - untrap(); - - static void - forward(int sig, siginfo_t *si, void *ucv); - - void - handle(int sig, siginfo_t *si, void *ucv); - + void trap(); + void untrap(); + static void forward(int sig, siginfo_t *si, void *ucv); + void handle(int sig, siginfo_t *si, void *ucv); public: SigBusHandler(StateFile *stateFile); - ~SigBusHandler(); - bool - fired() const - { - return _fired; - } + bool fired() const { return _fired; } /* * Setup siglongjmp based unwinding, used by unit tests. */ - void - setUnwind(sigjmp_buf *unwind) - { - _unwind = unwind; - } + void setUnwind(sigjmp_buf *unwind) { _unwind = unwind; } }; - } - diff --git a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.cpp b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.cpp index c86303e5a97..0f1648d7f67 100644 --- a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.cpp +++ b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "slime_output_raw_buf_adapter.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/util/statebuf.cpp b/searchlib/src/vespa/searchlib/util/statebuf.cpp index 69021f3b5f5..2a0f6e1db4a 100644 --- a/searchlib/src/vespa/searchlib/util/statebuf.cpp +++ b/searchlib/src/vespa/searchlib/util/statebuf.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "statebuf.h" static const char *hexx = "0123456789abcdef"; -namespace search -{ +namespace search { void StateBuf::overflow() noexcept diff --git a/searchlib/src/vespa/searchlib/util/statefile.cpp b/searchlib/src/vespa/searchlib/util/statefile.cpp index d093a036f19..58a2167f30f 100644 --- a/searchlib/src/vespa/searchlib/util/statefile.cpp +++ b/searchlib/src/vespa/searchlib/util/statefile.cpp @@ -1,18 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "statefile.h" +#include <vespa/vespalib/util/stringfmt.h> #include <system_error> #include <mutex> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> using Mutex = std::mutex; using Guard = std::lock_guard<Mutex>; -namespace search -{ +namespace search { -namespace -{ +namespace { Mutex stateMutex; @@ -456,5 +457,4 @@ StateFile::getGen() const return _gen; } - } diff --git a/searchlib/src/vespa/searchlib/util/statefile.h b/searchlib/src/vespa/searchlib/util/statefile.h index fddfc5bb8fc..95789c0b1bc 100644 --- a/searchlib/src/vespa/searchlib/util/statefile.h +++ b/searchlib/src/vespa/searchlib/util/statefile.h @@ -4,9 +4,9 @@ #include <string> #include <atomic> +#include <vector> -namespace search -{ +namespace search { /* * Class used to store application state in a mostly safe manner. @@ -30,27 +30,23 @@ class StateFile /* * Zero pad file, to ensure that a later write won't run out of space. */ - void - zeroPad(); + void zeroPad(); /* * Read state file to buffer in raw form, including padding. */ - void - readRawState(std::vector<char> &buf); + void readRawState(std::vector<char> &buf); /* * Trim padding and everything after state (i.e. stop at first newline). */ - static void - trimState(std::vector<char> &buf); + static void trimState(std::vector<char> &buf); /* * Trim partial state from end of history. */ - static void - trimHistory(std::vector<char> &history, const char *historyName, int hfd, - std::vector<char> &lastHistoryState); + static void trimHistory(std::vector<char> &history, const char *historyName, int hfd, + std::vector<char> &lastHistoryState); /* * Fixup history: trim partial state from end and append current state @@ -58,49 +54,36 @@ class StateFile * If main state file doesn't have a state but history has a state then * restore main state from history. */ - void - fixupHistory(); + void fixupHistory(); /* * Check that state doesn't contain nul bytes or early newline and * that it is terminated by a newline at end. */ - void - checkState(const char *buf, size_t bufLen) noexcept; - - void - internalAddSignalState(const char *buf, size_t bufLen, - const char *name, - int appendFlag, - const char *openerr, - const char *writeerr, - const char *fsyncerr, - const char *closeerr) noexcept; - - void - addSignalState(const char *buf, size_t bufLen) noexcept; + void checkState(const char *buf, size_t bufLen) noexcept; + + void internalAddSignalState(const char *buf, size_t bufLen, const char *name, int appendFlag, + const char *openerr, const char *writeerr, const char *fsyncerr, + const char *closeerr) noexcept; + + void addSignalState(const char *buf, size_t bufLen) noexcept; public: StateFile(const std::string &name); - ~StateFile(); - void - addState(const char *buf, size_t bufLen, bool signal); + void addState(const char *buf, size_t bufLen, bool signal); - static void - erase(const std::string &name); + static void erase(const std::string &name); /* * Read state file to buffer and trim it down to a state. */ - void - readState(std::vector<char> &buf); + void readState(std::vector<char> &buf); /* * Get current state generation (bumped whenever new state is written). */ - int - getGen(void) const; + int getGen() const; }; } diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index a8fb15f2eb1..82a2f35e738 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -2,20 +2,13 @@ // Copyright (C) 2000-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -/* - * Note for bugs / fixes: - * Please update fastserver4/test/regress/url/testurl.cpp - * with test for new url's when bugs are discovered/fixed. - */ +#include "url.h" -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> -#include <vespa/searchlib/util/url.h> - LOG_SETUP(".searchlib.util.url"); -namespace search { -namespace util { +namespace search::util { + bool URL::IsAlphaChar(unsigned char c) // According to RFC2396 { @@ -189,13 +182,13 @@ URL::URL(const unsigned char *url, size_t len) : _gotCompleteURL(false) { Reset(); - if (url != NULL) + if (url != nullptr) SetURL(url, len); } void -URL::Reset(void) +URL::Reset() { _gotCompleteURL = false; @@ -303,7 +296,7 @@ URL::SetURL(const unsigned char *url, size_t length) // Locate main-tld info. ptmp = reinterpret_cast<unsigned char *> (strrchr(reinterpret_cast<char *>(_host), '.')); - if (ptmp != NULL) { + if (ptmp != nullptr) { _maintld = &ptmp[1]; _startMainTld = _startHost + (_maintld - _host); if (*_tld == '\0') { @@ -315,7 +308,7 @@ URL::SetURL(const unsigned char *url, size_t length) if (_siteowner[0] == '\0') { pso = reinterpret_cast<unsigned char *> (strrchr(reinterpret_cast<char *>(_host), '.')); - if (pso != NULL && pso > _host) { + if (pso != nullptr && pso > _host) { pso--; solen = 0; while (pso > _host && *pso != '.') { @@ -367,7 +360,7 @@ URL::SetURL(const unsigned char *url, size_t length) extension = reinterpret_cast<unsigned char *> (strrchr(reinterpret_cast<char *>(_filename), '.')); - if (extension != NULL) { + if (extension != nullptr) { extension++; strcpy(reinterpret_cast<char *>(_extension), reinterpret_cast<char *>(extension)); @@ -376,7 +369,7 @@ URL::SetURL(const unsigned char *url, size_t length) // Parse params part. if ((ptmp = reinterpret_cast<unsigned char *> - (strchr(reinterpret_cast<char *>(_path), ';'))) != NULL) { + (strchr(reinterpret_cast<char *>(_path), ';'))) != nullptr) { ptmp++; _startParams = _startPath + (ptmp - _path); ParseURLPart<IsParamsChar>(ptmp, _params, sizeof(_params)); @@ -410,7 +403,7 @@ URL::SetURL(const unsigned char *url, size_t length) } bool -URL::IsBaseURL(void) const +URL::IsBaseURL() const { return (_scheme[0] != '\0' && _host[0] != '\0' && @@ -457,7 +450,7 @@ URL::GetToken(URL_CONTEXT &ctx) if (_token[0] != '\0') return _token; else - return NULL; + return nullptr; } const char * @@ -494,7 +487,7 @@ URL::ContextName(URL_CONTEXT ctx) } void -URL::Dump(void) +URL::Dump() { printf("URL: '%s'\n", _url); @@ -546,10 +539,9 @@ URL::Dump(void) const unsigned char *token; URL_CONTEXT ctx; - while ((token = GetToken(ctx)) != NULL) { + while ((token = GetToken(ctx)) != nullptr) { printf("TOKEN: %s '%s'\n", ContextName(ctx), token); } } } -} diff --git a/searchlib/src/vespa/searchlib/util/url.h b/searchlib/src/vespa/searchlib/util/url.h index e7f50b2e31a..88a4ef6d9f7 100644 --- a/searchlib/src/vespa/searchlib/util/url.h +++ b/searchlib/src/vespa/searchlib/util/url.h @@ -24,8 +24,9 @@ * roughly 40kB / instance. */ -namespace search { -namespace util { +#include <cstddef> + +namespace search::util { class URL { @@ -87,7 +88,7 @@ protected: bool _gotCompleteURL; - void Reset(void); + void Reset(); template <bool (*IsPartChar)(unsigned char c)> static unsigned char *ParseURLPart(unsigned char *url, @@ -138,7 +139,7 @@ public: * * @return true if this is an absolute URL, false otherwise. */ - bool IsBaseURL(void) const; + bool IsBaseURL() const; /** * Get a pointer to the current URL. @@ -252,8 +253,8 @@ public: /** * Get tokens with corresponding context information from the current url. * The first call to this function will return the first token in the url. - * This function may be called repetedly untill the value NULL is returned. - * @return Pointer to string containing the token, NULL when all tokens have + * This function may be called repetedly untill the value nullptr is returned. + * @return Pointer to string containing the token, nullptr when all tokens have * been returned. */ const unsigned char *GetToken(URL_CONTEXT &ctx); @@ -268,10 +269,7 @@ public: * Dump the contents of the URL and subelements to stdout. Only * elements that contains information are shown. */ - void Dump(void); + void Dump(); }; -} -} - - +}
\ No newline at end of file diff --git a/searchsummary/src/tests/docsumformat/docsum-parse.cpp b/searchsummary/src/tests/docsumformat/docsum-parse.cpp index 5fa7009464c..16cdcd1934e 100644 --- a/searchsummary/src/tests/docsumformat/docsum-parse.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-parse.cpp @@ -3,7 +3,6 @@ // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("docsum-parse"); #include <vespa/fnet/frt/frt.h> diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 8f900700d5d..7087b0350b3 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for positionsdfw. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("positionsdfw_test"); diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp index 59d949f40ca..8004b2dba53 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp @@ -2,11 +2,10 @@ // Copyright (C) 2001-2003 Fast Search & Transfer ASA // Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> +#include "extractkeywordstest.h" #include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/searchlib/parsequery/simplequerystack.h> -#include <vespa/searchlib/util/rawbuf.h> -#include "extractkeywordstest.h" +#include <vespa/fastos/time.h> #define NUMTESTS 5 diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h b/searchsummary/src/tests/extractkeywords/extractkeywordstest.h index 2b24ce39136..f0a5c93a6cb 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.h @@ -4,12 +4,9 @@ #pragma once +#include <vespa/fastos/app.h> -namespace search { -namespace docummary { -class KeywordExtractor; -} -} +namespace search::docsummary { class KeywordExtractor; } class ExtractKeywordsTest : public FastOS_Application { @@ -25,9 +22,8 @@ private: bool RunTest(int i, bool verify); public: - ExtractKeywordsTest(void) - : _extractor(NULL) - { - } + ExtractKeywordsTest() + : _extractor(nullptr) + {} }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumformat.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumformat.cpp index f964eafb2c9..baa2afb4631 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumformat.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumformat.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumformat.h" +#include <cassert> -namespace search { -namespace docsummary { +namespace search::docsummary { size_t DocsumFormat::addByte(search::RawBuf &target, uint8_t value) @@ -98,5 +98,4 @@ DocsumFormat::addEmpty(ResType type, search::RawBuf &target) return 0; } -} // namespace docsummary -} // namespace search +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 8760a12fb82..4e11bbcd979 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -2,17 +2,14 @@ #include "juniperdfw.h" #include "docsumwriter.h" -#include "docsumfieldwriter.h" #include "docsumstate.h" -#include "keywordextractor.h" -#include "docsumformat.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/queryeval/split_float.h> - #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/juniper/config.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.dynamicteaserdfw"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index e2dda337e73..697e2ed6722 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -6,12 +6,12 @@ #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <climits> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.geoposdfw"); -namespace search { -namespace docsummary { +namespace search::docsummary { using attribute::IAttributeVector; using attribute::IAttributeContext; @@ -110,5 +110,3 @@ GeoPositionDFW::create(const char *attribute_name, } -} - diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index 5b38a00fa3b..4b7e3c49cf5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -41,6 +41,28 @@ KeywordExtractor::~KeywordExtractor() } } +KeywordExtractor::IndexPrefix::IndexPrefix(const char *prefix, IndexPrefix **list) + : _prefix(NULL), + _prefixLen(0), + _next(NULL) +{ + _prefix = strdup(prefix); + assert(_prefix != NULL); + _prefixLen = strlen(prefix); + _next = *list; + *list = this; +} + +KeywordExtractor::IndexPrefix::~IndexPrefix() +{ + free(_prefix); +} + +bool +KeywordExtractor::IndexPrefix::Match(const char *idxName) const +{ + return (strncmp(idxName, _prefix, _prefixLen) == 0); +} void KeywordExtractor::AddLegalIndexSpec(const char *spec) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index a276b5bf38a..5ee598d98d9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -31,29 +31,9 @@ public: int _prefixLen; IndexPrefix *_next; - - IndexPrefix(const char *prefix, - IndexPrefix **list) - : _prefix(NULL), - _prefixLen(0), - _next(NULL) - { - _prefix = strdup(prefix); - assert(_prefix != NULL); - _prefixLen = strlen(prefix); - _next = *list; - *list = this; - } - - ~IndexPrefix() - { - free(_prefix); - } - - bool Match(const char *idxName) const - { - return (strncmp(idxName, _prefix, _prefixLen) == 0); - } + IndexPrefix(const char *prefix, IndexPrefix **list); + ~IndexPrefix(); + bool Match(const char *idxName) const; }; private: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguisticsannotation.cpp b/searchsummary/src/vespa/searchsummary/docsummary/linguisticsannotation.cpp index 124396aac5a..3513ab8918a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguisticsannotation.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguisticsannotation.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchsummary.docsummary.linguisticsannotation"); - #include "linguisticsannotation.h" #include <vespa/document/datatype/datatype.h> #include <vespa/document/datatype/primitivedatatype.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 841840d8349..be9517eba03 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -2,16 +2,15 @@ #include "positionsdfw.h" #include "docsumstate.h" -#include "idocsumenvironment.h" #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/stllike/asciistream.h> #include <cmath> +#include <climits> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.positionsdfw"); -namespace search { -namespace docsummary { +namespace search::docsummary { using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -223,5 +222,4 @@ AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, return ret; } -} // namespace docsummary -} // namespace search +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 9c28861eee6..ba63d2cd6a1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "resultclass.h" #include "resultconfig.h" +#include <cassert> #include <zlib.h> namespace search { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/searchdatatype.cpp b/searchsummary/src/vespa/searchsummary/docsummary/searchdatatype.cpp index 6e60628739a..0b8464c0100 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/searchdatatype.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/searchdatatype.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".search.docsummary.searchdatatype"); - #include "searchdatatype.h" #include <vespa/document/base/field.h> #include <vespa/document/datatype/primitivedatatype.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp index 0d3a212fc1f..d0beb2bd315 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp @@ -3,12 +3,12 @@ #include "urlresult.h" #include "resultconfig.h" #include <zlib.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.urlresult"); -namespace search { -namespace docsummary { +namespace search::docsummary { urlresult::urlresult(uint32_t partition, uint32_t docid, HitRank metric) : _partition(partition), @@ -811,4 +811,3 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } } -} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java index 83d6d6142e0..cacb2b73f45 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java @@ -17,7 +17,16 @@ import java.util.Set; */ class DimensionCache { - private final Map<String, LinkedHashMap<Point, UntypedMetric>> persistentData = new HashMap<>(); + private static class TimeStampedMetric { + public final long millis; + public final UntypedMetric metric; + public TimeStampedMetric(long millis, UntypedMetric metric) { + this.millis = millis; + this.metric = metric; + } + } + + private final Map<String, LinkedHashMap<Point, TimeStampedMetric>> persistentData = new HashMap<>(); private final int pointsToKeep; public DimensionCache(int pointsToKeep) { @@ -49,41 +58,49 @@ class DimensionCache { if (toDelete == null) { return; } + long millis = toDelete.gotTimeStamps ? toDelete.toMillis : System.currentTimeMillis(); for (Map.Entry<String, List<Entry<Point, UntypedMetric>>> metric : toDelete.getValuesByMetricName().entrySet()) { - LinkedHashMap<Point, UntypedMetric> cachedPoints = getCachedMetric(metric.getKey()); + LinkedHashMap<Point, TimeStampedMetric> cachedPoints = getCachedMetric(metric.getKey()); for (Entry<Point, UntypedMetric> newestInterval : metric.getValue()) { // overwriting an existing entry does not update the order // in the map cachedPoints.remove(newestInterval.getKey()); - cachedPoints.put(newestInterval.getKey(), newestInterval.getValue()); + TimeStampedMetric toInsert = new TimeStampedMetric(millis, newestInterval.getValue()); + cachedPoints.put(newestInterval.getKey(), toInsert); } } } + private static final long MAX_AGE_MILLIS = 4 * 3600 * 1000; + private void padMetric(String metric, Bucket toPresent, int currentDataPoints) { - final LinkedHashMap<Point, UntypedMetric> cachedPoints = getCachedMetric(metric); + final LinkedHashMap<Point, TimeStampedMetric> cachedPoints = getCachedMetric(metric); int toAdd = pointsToKeep - currentDataPoints; @SuppressWarnings({"unchecked","rawtypes"}) - Entry<Point, UntypedMetric>[] cachedEntries = cachedPoints.entrySet().toArray(new Entry[0]); + Entry<Point, TimeStampedMetric>[] cachedEntries = cachedPoints.entrySet().toArray(new Entry[0]); + long nowMillis = System.currentTimeMillis(); for (int i = cachedEntries.length - 1; i >= 0 && toAdd > 0; --i) { - Entry<Point, UntypedMetric> leastOld = cachedEntries[i]; + Entry<Point, TimeStampedMetric> leastOld = cachedEntries[i]; + if (leastOld.getValue().millis + MAX_AGE_MILLIS < nowMillis) { + continue; + } final Identifier id = new Identifier(metric, leastOld.getKey()); if (!toPresent.hasIdentifier(id)) { - toPresent.put(id, leastOld.getValue().pruneData()); + toPresent.put(id, leastOld.getValue().metric.pruneData()); --toAdd; } } } @SuppressWarnings("serial") - private LinkedHashMap<Point, UntypedMetric> getCachedMetric(String metricName) { - LinkedHashMap<Point, UntypedMetric> points = persistentData.get(metricName); + private LinkedHashMap<Point, TimeStampedMetric> getCachedMetric(String metricName) { + LinkedHashMap<Point, TimeStampedMetric> points = persistentData.get(metricName); if (points == null) { - points = new LinkedHashMap<Point, UntypedMetric>(16, 0.75f, false) { - protected boolean removeEldestEntry(Map.Entry<Point, UntypedMetric> eldest) { + points = new LinkedHashMap<Point, TimeStampedMetric>(16, 0.75f, false) { + protected @Override boolean removeEldestEntry(Map.Entry<Point, TimeStampedMetric> eldest) { return size() > pointsToKeep; } }; diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java index c73493f7570..671f8441016 100644 --- a/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java @@ -80,6 +80,22 @@ public class DimensionsCacheTest { } @Test + public final void requireThatOldDataIsForgotten() { + Bucket first = new Bucket(); // "now" as timestamp + populateDimensionLessValue("one", first, 2); + cache.updateDimensionPersistence(first, new Bucket()); + Bucket second = new Bucket(17, 42); // really old timestamp + populateDimensionLessValue("other", second, 3); + Bucket third = new Bucket(); + populateDimensionLessValue("two", third, 4); + cache.updateDimensionPersistence(second, third); + Collection<String> names = third.getAllMetricNames(); + assertEquals(2, names.size()); + assertTrue(names.contains("one")); + assertTrue(names.contains("two")); + } + + @Test public final void testUpdateWithNullThenDataThenNoDataThenData() { Bucket first = new Bucket(); Bucket second = new Bucket(); diff --git a/slobrok/src/apps/check_slobrok/check_slobrok.cpp b/slobrok/src/apps/check_slobrok/check_slobrok.cpp index 9d588eab39f..f2b6fa0e71b 100644 --- a/slobrok/src/apps/check_slobrok/check_slobrok.cpp +++ b/slobrok/src/apps/check_slobrok/check_slobrok.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fnet/frt/frt.h> #include <vespa/fastos/app.h> -#include <string> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> + #include <sstream> #include <vespa/log/log.h> diff --git a/slobrok/src/apps/sbcmd/sbcmd.cpp b/slobrok/src/apps/sbcmd/sbcmd.cpp index a96c4609137..927fab6ebff 100644 --- a/slobrok/src/apps/sbcmd/sbcmd.cpp +++ b/slobrok/src/apps/sbcmd/sbcmd.cpp @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fnet/frt/frt.h> #include <vespa/fastos/app.h> -#include <string> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/frt/rpcrequest.h> #include <sstream> #include <vespa/log/log.h> diff --git a/slobrok/src/tests/backoff/testbackoff.cpp b/slobrok/src/tests/backoff/testbackoff.cpp index af0dbd5bc81..f64ca79917c 100644 --- a/slobrok/src/tests/backoff/testbackoff.cpp +++ b/slobrok/src/tests/backoff/testbackoff.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("backoff_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/slobrok/backoff.h> -#include <algorithm> + +#include <vespa/log/log.h> +LOG_SETUP("backoff_test"); using slobrok::api::BackOff; diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index e8977f678e4..086917dd5d7 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -1,17 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("configure_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> #include <vespa/config/config.h> #include <vespa/config-slobroks.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/vespalib/util/host_name.h> #include <sstream> #include <algorithm> #include <iostream> -#include <vespa/vespalib/util/host_name.h> + +#include <vespa/log/log.h> +LOG_SETUP("configure_test"); using slobrok::api::MirrorAPI; using slobrok::api::RegisterAPI; diff --git a/slobrok/src/tests/configure/gencfg.cpp b/slobrok/src/tests/configure/gencfg.cpp index 85cc8629830..8e0de4abb8c 100644 --- a/slobrok/src/tests/configure/gencfg.cpp +++ b/slobrok/src/tests/configure/gencfg.cpp @@ -1,9 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> #include <vespa/vespalib/util/host_name.h> +#include <cstdio> int main(int, char **) diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 4370931b785..4a1f4f023ca 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -1,12 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("mirrorapi_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/config-slobroks.h> #include <algorithm> #include <vespa/slobrok/server/slobrokserver.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + +#include <vespa/log/log.h> +LOG_SETUP("mirrorapi_test"); using slobrok::api::MirrorAPI; using slobrok::SlobrokServer; diff --git a/slobrok/src/tests/oldapi/mirror.cpp b/slobrok/src/tests/oldapi/mirror.cpp index 22304091395..fb9fa167c32 100644 --- a/slobrok/src/tests/oldapi/mirror.cpp +++ b/slobrok/src/tests/oldapi/mirror.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "mirror.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + #include <vespa/log/log.h> LOG_SETUP(".slobrok.mirror"); -#include <vespa/fastos/fastos.h> -#include <vespa/fnet/frt/frt.h> -#include "mirror.h" -#include <memory> -namespace slobrok { -namespace api { +namespace slobrok::api { MirrorOld::MirrorOld(FRT_Supervisor &orb, const std::vector<std::string> &slobroks) @@ -171,5 +170,4 @@ MirrorOld::RequestDone(FRT_RPCRequest *req) ScheduleNow(); } -} // namespace api -} // namespace slobrok +} // namespace slobrok::api diff --git a/slobrok/src/tests/oldapi/mirror.h b/slobrok/src/tests/oldapi/mirror.h index b83392dec5e..b6259c76b7e 100644 --- a/slobrok/src/tests/oldapi/mirror.h +++ b/slobrok/src/tests/oldapi/mirror.h @@ -1,12 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/gencnt.h> #include <vespa/slobrok/backoff.h> +#include <vespa/fnet/frt/invoker.h> -namespace slobrok { -namespace api { +class FRT_Target; + +namespace slobrok::api { /** * @brief Defines an interface for the name server lookup. @@ -122,5 +123,4 @@ private: FRT_RPCRequest *_req; }; -} // namespace api -} // namespace slobrok +} // namespace slobrok::api diff --git a/slobrok/src/tests/oldapi/old.cpp b/slobrok/src/tests/oldapi/old.cpp index 8b620cda3b2..23cde98674f 100644 --- a/slobrok/src/tests/oldapi/old.cpp +++ b/slobrok/src/tests/oldapi/old.cpp @@ -1,12 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("mirrorapi_test"); #include <vespa/vespalib/testkit/testapp.h> #include "mirror.h" #include <vespa/config-slobroks.h> -#include <algorithm> #include <vespa/slobrok/server/slobrokserver.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + +#include <vespa/log/log.h> +LOG_SETUP("mirrorapi_test"); using slobrok::api::MirrorOld; using slobrok::SlobrokServer; diff --git a/slobrok/src/tests/registerapi/registerapi.cpp b/slobrok/src/tests/registerapi/registerapi.cpp index 8e4cb2404f9..e61a28e3c28 100644 --- a/slobrok/src/tests/registerapi/registerapi.cpp +++ b/slobrok/src/tests/registerapi/registerapi.cpp @@ -1,15 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("registerapi_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> #include <vespa/slobrok/server/slobrokserver.h> +#include <vespa/fnet/frt/supervisor.h> #include <sstream> #include <algorithm> +#include <vespa/log/log.h> +LOG_SETUP("registerapi_test"); + using slobrok::api::MirrorAPI; using slobrok::api::RegisterAPI; using slobrok::SlobrokServer; diff --git a/slobrok/src/tests/standalone/standalone.cpp b/slobrok/src/tests/standalone/standalone.cpp index 7cae6eabbad..9d9ccbdeadd 100644 --- a/slobrok/src/tests/standalone/standalone.cpp +++ b/slobrok/src/tests/standalone/standalone.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/server/slobrokserver.h> -#include <string> - +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> //----------------------------------------------------------------------------- diff --git a/slobrok/src/tests/startsome/rpc_info.cpp b/slobrok/src/tests/startsome/rpc_info.cpp index be5418ff1a8..40e4db46bf5 100644 --- a/slobrok/src/tests/startsome/rpc_info.cpp +++ b/slobrok/src/tests/startsome/rpc_info.cpp @@ -1,7 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fnet/frt/frt.h> #include <vespa/fastos/app.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/target.h> + class RPCInfo : public FastOS_Application { diff --git a/slobrok/src/tests/startsome/tstdst.cpp b/slobrok/src/tests/startsome/tstdst.cpp index 7d3fcef8695..b645f79efa4 100644 --- a/slobrok/src/tests/startsome/tstdst.cpp +++ b/slobrok/src/tests/startsome/tstdst.cpp @@ -1,8 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/host_name.h> -#include <vespa/fnet/frt/frt.h> #include <vespa/fastos/app.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/invoker.h> +#include <vespa/fnet/transport.h> +#include <vespa/fnet/frt/target.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("testrpcserver"); diff --git a/slobrok/src/vespa/slobrok/cfg.cpp b/slobrok/src/vespa/slobrok/cfg.cpp index 50320f7dd23..733377d5e0c 100644 --- a/slobrok/src/vespa/slobrok/cfg.cpp +++ b/slobrok/src/vespa/slobrok/cfg.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "cfg.h" -#include <vespa/log/log.h> -#include <vespa/vespalib/util/sync.h> +#include <vespa/log/log.h> LOG_SETUP(".slobrok.configurator"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/sblist.cpp b/slobrok/src/vespa/slobrok/sblist.cpp index be702c6c5c2..de7795164d6 100644 --- a/slobrok/src/vespa/slobrok/sblist.cpp +++ b/slobrok/src/vespa/slobrok/sblist.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "sblist.h" -#include <vespa/log/log.h> -LOG_SETUP(".slobrok.list"); #include <vespa/vespalib/util/random.h> +#include <vespa/log/log.h> +LOG_SETUP(".slobrok.list"); using vespalib::LockGuard; -namespace slobrok { -namespace api { +namespace slobrok::api { SlobrokList::SlobrokList() : _lock(), @@ -96,6 +94,4 @@ SlobrokList::setup(const std::vector<std::string> &specList) } } - -} // namespace api -} // namespace slobrok +} // namespace slobrok::api diff --git a/slobrok/src/vespa/slobrok/sblist.h b/slobrok/src/vespa/slobrok/sblist.h index 53962363609..81bf5de325d 100644 --- a/slobrok/src/vespa/slobrok/sblist.h +++ b/slobrok/src/vespa/slobrok/sblist.h @@ -4,8 +4,7 @@ #include "cfg.h" #include <vespa/vespalib/util/sync.h> -namespace slobrok { -namespace api { +namespace slobrok::api { /** * @brief List of connection specs for service location brokers @@ -50,6 +49,4 @@ private: size_t _retryCount; }; -} // namespace api -} // namespace slobrok - +} diff --git a/slobrok/src/vespa/slobrok/sbmirror.cpp b/slobrok/src/vespa/slobrok/sbmirror.cpp index 7b0974d5b0d..d828f64f1c1 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.cpp +++ b/slobrok/src/vespa/slobrok/sbmirror.cpp @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "sbmirror.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + #include <vespa/log/log.h> LOG_SETUP(".slobrok.mirror"); -#include <vespa/fnet/frt/frt.h> -#include <vespa/config-slobroks.h> -#include "sbmirror.h" -#include <memory> using vespalib::LockGuard; diff --git a/slobrok/src/vespa/slobrok/sbmirror.h b/slobrok/src/vespa/slobrok/sbmirror.h index c9bca70885f..002cd43929f 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.h +++ b/slobrok/src/vespa/slobrok/sbmirror.h @@ -4,13 +4,12 @@ #include "imirrorapi.h" #include "backoff.h" #include "sblist.h" -#include "cfg.h" -#include <vespa/fnet/frt/frt.h> #include <vespa/vespalib/util/gencnt.h> -#include <vespa/vespalib/util/sync.h> +#include <vespa/fnet/frt/invoker.h> -namespace slobrok { -namespace api { +class FRT_Target; + +namespace slobrok::api { /** * @brief A MirrorAPI object is used to keep track of the services registered @@ -108,6 +107,4 @@ private: FRT_RPCRequest *_req; }; -} // namespace api -} // namespace slobrok - +} // namespace slobrok::api diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp index a904a49884a..0d1e506b613 100644 --- a/slobrok/src/vespa/slobrok/sbregister.cpp +++ b/slobrok/src/vespa/slobrok/sbregister.cpp @@ -1,15 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fnet/frt/frt.h> -#include <vespa/config-slobroks.h> #include "sbregister.h" -#include <memory> -#include <sstream> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/exceptions.h> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".slobrok.register"); +using vespalib::NetworkSetupFailureException; + namespace { vespalib::string @@ -47,8 +50,7 @@ discard(std::vector<vespalib::string> &vec, const vespalib::stringref & val) } // namespace <unnamed> -namespace slobrok { -namespace api { +namespace slobrok::api { RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config) : FNET_Task(orb.GetScheduler()), @@ -69,7 +71,10 @@ RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config _req(0) { _configurator->poll(); - LOG_ASSERT(_slobrokSpecs.ok()); + if ( ! _slobrokSpecs.ok()) { + throw NetworkSetupFailureException("Failed configuring the RegisterAPI. No valid slobrok specs from config", + VESPA_STRLOC); + } ScheduleNow(); } @@ -127,8 +132,8 @@ RegisterAPI::handleReqDone() _reqDone = false; if (_req->IsError()) { if (_req->GetErrorCode() != FRTE_RPC_METHOD_FAILED) { - LOG(debug, "register failed: %s (code %d)", - _req->GetErrorMessage(), _req->GetErrorCode()); + LOG(debug, "register failed: %s (code %d)", + _req->GetErrorMessage(), _req->GetErrorCode()); // unexpected error; close our connection to this // slobrok server and try again with a fresh slate if (_target != 0) { @@ -187,7 +192,7 @@ RegisterAPI::handleReconnect() LOG(warning, "cannot connect to location broker at %s " "(retry in %f seconds)", cps.c_str(), delay); } else { - LOG(debug, "slobrok retry in %f seconds", delay); + LOG(debug, "slobrok retry in %f seconds", delay); } return; } @@ -220,7 +225,7 @@ RegisterAPI::handlePending() _req = _orb.AllocRPCRequest(); _req->SetMethodName("slobrok.unregisterRpcServer"); _req->GetParams()->AddString(name.c_str()); - LOG(debug, "unregister [%s]", name.c_str()); + LOG(debug, "unregister [%s]", name.c_str()); _req->GetParams()->AddString(createSpec(_orb).c_str()); _target->InvokeAsync(_req, 35.0, this); } else if (reg) { @@ -228,7 +233,7 @@ RegisterAPI::handlePending() _req = _orb.AllocRPCRequest(); _req->SetMethodName("slobrok.registerRpcServer"); _req->GetParams()->AddString(name.c_str()); - LOG(debug, "register [%s]", name.c_str()); + LOG(debug, "register [%s]", name.c_str()); _req->GetParams()->AddString(createSpec(_orb).c_str()); _target->InvokeAsync(_req, 35.0, this); } else { @@ -236,7 +241,7 @@ RegisterAPI::handlePending() // names after a long delay. _lock.Lock(); _pending = _names; - LOG(debug, "done, reschedule in 30s"); + LOG(debug, "done, reschedule in 30s"); _busy = false; Schedule(30.0); _lock.Unlock(); @@ -248,7 +253,7 @@ RegisterAPI::PerformTask() { handleReqDone(); if (_req != 0) { - LOG(debug, "req in progress"); + LOG(debug, "req in progress"); return; // current request still in progress, don't start anything new } handleReconnect(); @@ -312,5 +317,4 @@ RegisterAPI::RPCHooks::rpc_notifyUnregistered(FRT_RPCRequest *req) LOG(warning, "unregistered name %s", args[0]._string._str); } -} // namespace api -} // namespace slobrok +} diff --git a/slobrok/src/vespa/slobrok/sbregister.h b/slobrok/src/vespa/slobrok/sbregister.h index 7a71db8d480..be6516ecdbc 100644 --- a/slobrok/src/vespa/slobrok/sbregister.h +++ b/slobrok/src/vespa/slobrok/sbregister.h @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> -#include <vespa/vespalib/util/gencnt.h> #include "backoff.h" #include "sblist.h" #include "cfg.h" -#include <string> -#include <vector> +#include <vespa/fnet/frt/invoker.h> +#include <vespa/fnet/frt/invokable.h> -namespace slobrok { -namespace api { +class FRT_Target; + +namespace slobrok::api { /** * @brief A RegisterAPI object is used to register and unregister @@ -100,6 +99,4 @@ private: FRT_RPCRequest *_req; }; -} // namespace api -} // namespace slobrok - +} // namespace slobrok::api diff --git a/slobrok/src/vespa/slobrok/server/cmd.cpp b/slobrok/src/vespa/slobrok/server/cmd.cpp index b83bcbe2fba..c22c9cdaebd 100644 --- a/slobrok/src/vespa/slobrok/server/cmd.cpp +++ b/slobrok/src/vespa/slobrok/server/cmd.cpp @@ -1,20 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".cmd"); - -#include <vespa/fnet/frt/frt.h> #include "cmd.h" -#include "ok_state.h" -#include "named_service.h" -#include "reserved_name.h" #include "rpc_server_map.h" -#include "rpc_server_manager.h" +#include "reserved_name.h" #include "remote_slobrok.h" #include "sbenv.h" +#include <vespa/log/log.h> +LOG_SETUP(".cmd"); + namespace slobrok { //----------------------------------------------------------------------------- diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp index 197a1dd8a18..b4086f03527 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.cpp @@ -1,22 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - -#include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); - -#include <vector> -#include <deque> -#include <stdlib.h> #include "exchange_manager.h" - -#include "ok_state.h" -#include "named_service.h" #include "rpc_server_map.h" -#include "rpc_server_manager.h" -#include "remote_slobrok.h" #include "sbenv.h" -#include "cmd.h" +#include <vespa/fnet/frt/supervisor.h> + +#include <vespa/log/log.h> +LOG_SETUP(".rpcserver"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/exchange_manager.h b/slobrok/src/vespa/slobrok/server/exchange_manager.h index 20cf03e921a..b3883ffd92d 100644 --- a/slobrok/src/vespa/slobrok/server/exchange_manager.h +++ b/slobrok/src/vespa/slobrok/server/exchange_manager.h @@ -4,8 +4,6 @@ #include <deque> #include <string> -#include <vespa/fnet/frt/frt.h> - #include <vespa/vespalib/util/hashmap.h> #include "ok_state.h" #include "cmd.h" diff --git a/slobrok/src/vespa/slobrok/server/history.cpp b/slobrok/src/vespa/slobrok/server/history.cpp index fa2bf9c9023..f542d718b1f 100644 --- a/slobrok/src/vespa/slobrok/server/history.cpp +++ b/slobrok/src/vespa/slobrok/server/history.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "history.h" #include <vespa/log/log.h> LOG_SETUP(".history"); -#include "history.h" - namespace slobrok { void diff --git a/slobrok/src/vespa/slobrok/server/history.h b/slobrok/src/vespa/slobrok/server/history.h index 9898d5b21cc..9b0e629f7ab 100644 --- a/slobrok/src/vespa/slobrok/server/history.h +++ b/slobrok/src/vespa/slobrok/server/history.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/util/gencnt.h> -#include <vespa/vespalib/util/hashmap.h> #include <vector> #include <string> #include <set> diff --git a/slobrok/src/vespa/slobrok/server/i_monitored_server.h b/slobrok/src/vespa/slobrok/server/i_monitored_server.h index 1bfe52387e0..bc66844e4cf 100644 --- a/slobrok/src/vespa/slobrok/server/i_monitored_server.h +++ b/slobrok/src/vespa/slobrok/server/i_monitored_server.h @@ -1,8 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> - namespace slobrok { //----------------------------------------------------------------------------- diff --git a/slobrok/src/vespa/slobrok/server/i_rpc_server_manager.h b/slobrok/src/vespa/slobrok/server/i_rpc_server_manager.h index b093acd7c2a..bca885661b9 100644 --- a/slobrok/src/vespa/slobrok/server/i_rpc_server_manager.h +++ b/slobrok/src/vespa/slobrok/server/i_rpc_server_manager.h @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> - -#include "monitor.h" - #include <string> +class FRT_Supervisor; + namespace slobrok { class ManagedRpcServer; diff --git a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp index 0513f30c1dc..1003c239150 100644 --- a/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp +++ b/slobrok/src/vespa/slobrok/server/managed_rpc_server.cpp @@ -2,6 +2,9 @@ #include "managed_rpc_server.h" #include "i_rpc_server_manager.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> + #include <vespa/log/log.h> LOG_SETUP(".rpcserver"); diff --git a/slobrok/src/vespa/slobrok/server/managed_rpc_server.h b/slobrok/src/vespa/slobrok/server/managed_rpc_server.h index 4ade368e623..9b75671f1e2 100644 --- a/slobrok/src/vespa/slobrok/server/managed_rpc_server.h +++ b/slobrok/src/vespa/slobrok/server/managed_rpc_server.h @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> - -#include "monitor.h" #include "named_service.h" - -#include <string> +#include "monitor.h" +#include <vespa/fnet/frt/invoker.h> namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp index 09af3eca194..8752161505b 100644 --- a/slobrok/src/vespa/slobrok/server/metrics_producer.cpp +++ b/slobrok/src/vespa/slobrok/server/metrics_producer.cpp @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "metrics_producer.h" #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/fnet/task.h> +#include <vespa/fnet/transport.h> namespace slobrok { @@ -74,6 +77,9 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr, uint32_t prevTime, uint32_t currTime) { MetricSnapshot snapshot(prevTime, currTime); + snapshot.addCount("slobrok.heartbeats.failed", + "count of failed heartbeat requests", + curr.heartBeatFails - prev.heartBeatFails); snapshot.addCount("slobrok.requests.register", "count of register requests received", curr.registerReqs - prev.registerReqs); @@ -92,7 +98,7 @@ makeSnapshot(const RPCHooks::Metrics &prev, const RPCHooks::Metrics &curr, MetricsProducer::MetricsProducer(const RPCHooks &hooks, FNET_Transport &transport) : _rpcHooks(hooks), - _lastMetrics{ 0, 0, 0, 0, 0, 0, 0}, + _lastMetrics(RPCHooks::Metrics::zero()), _producer(), _startTime(time(NULL)), _lastSnapshotStart(_startTime), @@ -114,7 +120,7 @@ MetricsProducer::getTotalMetrics(const vespalib::string &) { uint32_t now = time(NULL); RPCHooks::Metrics current = _rpcHooks.getMetrics(); - RPCHooks::Metrics start{0, 0, 0, 0, 0, 0, 0}; + RPCHooks::Metrics start = RPCHooks::Metrics::zero(); return makeSnapshot(start, current, _startTime, now); } diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.h b/slobrok/src/vespa/slobrok/server/metrics_producer.h index d66a1d53071..65505a38c43 100644 --- a/slobrok/src/vespa/slobrok/server/metrics_producer.h +++ b/slobrok/src/vespa/slobrok/server/metrics_producer.h @@ -4,7 +4,8 @@ #include "rpchooks.h" #include <vespa/vespalib/net/metrics_producer.h> #include <vespa/vespalib/net/simple_metrics_producer.h> -#include <vespa/fnet/frt/frt.h> + +class FNET_Transport; namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/monitor.cpp b/slobrok/src/vespa/slobrok/server/monitor.cpp index 70df443af86..20afeb5ae23 100644 --- a/slobrok/src/vespa/slobrok/server/monitor.cpp +++ b/slobrok/src/vespa/slobrok/server/monitor.cpp @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "monitor.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/channel.h> namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/monitor.h b/slobrok/src/vespa/slobrok/server/monitor.h index a71c6b69821..7d55a4ad4ee 100644 --- a/slobrok/src/vespa/slobrok/server/monitor.h +++ b/slobrok/src/vespa/slobrok/server/monitor.h @@ -1,9 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> - #include "i_monitored_server.h" +#include <vespa/fnet/task.h> +#include <vespa/fnet/ipackethandler.h> + +class FRT_Supervisor; +class FRT_Target; namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/named_service.cpp b/slobrok/src/vespa/slobrok/server/named_service.cpp index f46acd700d2..33f0815772d 100644 --- a/slobrok/src/vespa/slobrok/server/named_service.cpp +++ b/slobrok/src/vespa/slobrok/server/named_service.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "named_service.h" #include <vespa/log/log.h> LOG_SETUP(".rpcserver"); -#include "named_service.h" -#include "i_rpc_server_manager.h" - namespace slobrok { //----------------------------------------------------------------------------- diff --git a/slobrok/src/vespa/slobrok/server/named_service.h b/slobrok/src/vespa/slobrok/server/named_service.h index 9d04475573a..db48bb6b80e 100644 --- a/slobrok/src/vespa/slobrok/server/named_service.h +++ b/slobrok/src/vespa/slobrok/server/named_service.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> #include <string> namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/random.h b/slobrok/src/vespa/slobrok/server/random.h index 0ab2597dd03..e8315db85a1 100644 --- a/slobrok/src/vespa/slobrok/server/random.h +++ b/slobrok/src/vespa/slobrok/server/random.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <stdlib.h> +#include <cstdlib> namespace { diff --git a/slobrok/src/vespa/slobrok/server/remote_check.cpp b/slobrok/src/vespa/slobrok/server/remote_check.cpp index 96360daf873..eac452db88c 100644 --- a/slobrok/src/vespa/slobrok/server/remote_check.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_check.cpp @@ -1,22 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - -#include <vespa/log/log.h> -LOG_SETUP(".remcheck"); - -#include <vespa/fnet/frt/frt.h> #include "remote_check.h" -#include "ok_state.h" #include "named_service.h" #include "rpc_server_map.h" #include "rpc_server_manager.h" #include "remote_slobrok.h" -#include "sbenv.h" #include "random.h" +#include "exchange_manager.h" -namespace slobrok { +#include <vespa/log/log.h> +LOG_SETUP(".remcheck"); +namespace slobrok { RemoteCheck::RemoteCheck(FNET_Scheduler *sched, RpcServerMap& rpcsrvmap, diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp index 23b8fc75e50..35f2405f82a 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.cpp @@ -2,7 +2,11 @@ #include "remote_slobrok.h" #include "rpc_server_map.h" +#include "rpc_server_manager.h" +#include "exchange_manager.h" #include "sbenv.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> #include <vespa/log/log.h> LOG_SETUP(".rpcserver"); diff --git a/slobrok/src/vespa/slobrok/server/remote_slobrok.h b/slobrok/src/vespa/slobrok/server/remote_slobrok.h index fdc4c6e492b..3989ab70237 100644 --- a/slobrok/src/vespa/slobrok/server/remote_slobrok.h +++ b/slobrok/src/vespa/slobrok/server/remote_slobrok.h @@ -1,29 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <deque> -#include <string> - -#include <vespa/fnet/frt/frt.h> - -#include <vespa/vespalib/util/hashmap.h> #include "ok_state.h" #include "cmd.h" #include "i_rpc_server_manager.h" -#include "rpc_server_manager.h" #include "managed_rpc_server.h" +#include <deque> namespace slobrok { //----------------------------------------------------------------------------- -class SBEnv; -class RpcServerMap; class RpcServerManager; class ExchangeManager; -using vespalib::HashMap; - //----------------------------------------------------------------------------- /** diff --git a/slobrok/src/vespa/slobrok/server/reserved_name.h b/slobrok/src/vespa/slobrok/server/reserved_name.h index b36e8e8ac50..8399c93380e 100644 --- a/slobrok/src/vespa/slobrok/server/reserved_name.h +++ b/slobrok/src/vespa/slobrok/server/reserved_name.h @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> -#include <string> - #include "named_service.h" +#include <vespa/fastos/time.h> namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp index f8d0a4a3897..9d34d29fc0b 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp @@ -1,20 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - -#include <vespa/log/log.h> -LOG_SETUP(".rpcserver"); - -#include <string> -#include <sstream> -#include <vespa/vespalib/util/stringfmt.h> #include "rpc_server_manager.h" -#include "ok_state.h" -#include "named_service.h" #include "reserved_name.h" #include "rpc_server_map.h" #include "remote_slobrok.h" #include "sbenv.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <sstream> + +#include <vespa/log/log.h> +LOG_SETUP(".rpcserver"); namespace slobrok { @@ -310,6 +305,7 @@ RpcServerManager::PerformTask() void RpcServerManager::notifyFailedRpcSrv(ManagedRpcServer *rpcsrv, std::string errmsg) { + _env.countFailedHeartbeat(); bool logged = false; NamedService *old = _rpcsrvmap.lookup(rpcsrv->getName()); if (old == rpcsrv) { diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.h b/slobrok/src/vespa/slobrok/server/rpc_server_manager.h index b355c39c77b..7866681feb1 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.h +++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.h @@ -1,17 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string> -#include <vector> - -#include <vespa/vespalib/util/hashmap.h> - -#include <vespa/fnet/frt/frt.h> - #include "ok_state.h" #include "cmd.h" #include "i_rpc_server_manager.h" #include "named_service.h" +#include <vespa/fnet/task.h> +#include <vector> namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp index 7c354874dc7..cf4ad69b819 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp @@ -1,17 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - -#include <vespa/log/log.h> -LOG_SETUP(".rpcsrvmap"); - -#include <vector> #include "rpc_server_map.h" -#include "named_service.h" #include "reserved_name.h" #include "rpc_server_manager.h" #include "sbenv.h" +#include <vespa/log/log.h> +LOG_SETUP(".rpcsrvmap"); + namespace slobrok { //----------------------------------------------------------------------------- diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.h b/slobrok/src/vespa/slobrok/server/rpc_server_map.h index 5f08039057a..3cadab0632d 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.h +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.h @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> -#include <vespa/vespalib/util/hashmap.h> -#include <vector> - #include "visible_map.h" namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 2eeafe7bbda..b3d8df33f6b 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -8,6 +8,8 @@ #include "remote_slobrok.h" #include "sbenv.h" #include "rpcmirror.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/vespalib/component/vtag.h> #include <vespa/log/log.h> LOG_SETUP(".rpchooks"); @@ -46,7 +48,7 @@ RPCHooks::RPCHooks(SBEnv &env, RpcServerMap& rpcsrvmap, RpcServerManager& rpcsrvman) : _env(env), _rpcsrvmap(rpcsrvmap), _rpcsrvmanager(rpcsrvman), - _cnts{0, 0, 0, 0, 0, 0, 0}, + _cnts(Metrics::zero()), _m_reporter(nullptr) { } @@ -60,6 +62,7 @@ RPCHooks::~RPCHooks() void RPCHooks::reportMetrics() { + EV_COUNT("heartbeats_failed", _cnts.heartBeatFails); EV_COUNT("register_reqs", _cnts.registerReqs); EV_COUNT("mirror_reqs", _cnts.mirrorReqs); EV_COUNT("wantadd_reqs", _cnts.wantAddReqs); diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h index 0f560d62975..6a8fb1e0e9a 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.h +++ b/slobrok/src/vespa/slobrok/server/rpchooks.h @@ -1,7 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> +#include <vespa/fnet/frt/invokable.h> + +class FNET_Task; +class FRT_Supervisor; namespace slobrok { @@ -22,6 +25,7 @@ class RPCHooks : public FRT_Invokable public: struct Metrics { + unsigned long heartBeatFails; unsigned long registerReqs; unsigned long mirrorReqs; unsigned long wantAddReqs; @@ -29,6 +33,7 @@ public: unsigned long doRemoveReqs; unsigned long adminReqs; unsigned long otherReqs; + static Metrics zero() { return Metrics{0,0,0,0,0,0,0,0}; } }; private: @@ -51,6 +56,7 @@ public: void initRPC(FRT_Supervisor *supervisor); void reportMetrics(); const Metrics& getMetrics() const { return _cnts; } + void countFailedHeartbeat() { _cnts.heartBeatFails++; } private: void rpc_lookupRpcServer(FRT_RPCRequest *req); diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp index 1ea6e3b4742..7c9d04fa9e5 100644 --- a/slobrok/src/vespa/slobrok/server/rpcmirror.cpp +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpcmirror.h" -#include <vespa/log/log.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/log/log.h> LOG_SETUP(".rpcmirror"); namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/rpcmirror.h b/slobrok/src/vespa/slobrok/server/rpcmirror.h index 1bb31720db5..45458d646e5 100644 --- a/slobrok/src/vespa/slobrok/server/rpcmirror.h +++ b/slobrok/src/vespa/slobrok/server/rpcmirror.h @@ -2,6 +2,10 @@ #pragma once #include "visible_map.h" +#include <vespa/fnet/task.h> + +class FRT_RPCRequest; +class FRT_Supervisor; namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 0cc0b2467e2..88d34f228c4 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -7,6 +7,8 @@ #include <vespa/vespalib/net/state_server.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/transport.h> #include <vespa/log/log.h> LOG_SETUP(".sbenv"); @@ -105,6 +107,10 @@ SBEnv::~SBEnv() getTransport()->WaitFinished(); } +FNET_Scheduler * +SBEnv::getScheduler() { + return _transport->GetScheduler(); +} void SBEnv::shutdown() diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index 009fa6c6fa7..0d9a89c45d0 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -64,7 +64,7 @@ public: ~SBEnv(); FNET_Transport *getTransport() { return _transport.get(); } - FNET_Scheduler *getScheduler() { return _transport->GetScheduler(); } + FNET_Scheduler *getScheduler(); FRT_Supervisor *getSupervisor() { return _supervisor.get(); } void shutdown(); @@ -85,6 +85,7 @@ public: OkState addPeer(const std::string& name, const std::string &spec); OkState removePeer(const std::string& name, const std::string &spec); + void countFailedHeartbeat() { _rpcHooks.countFailedHeartbeat(); } }; } // namespace slobrok diff --git a/slobrok/src/vespa/slobrok/server/selfcheck.cpp b/slobrok/src/vespa/slobrok/server/selfcheck.cpp index bb8cb930bae..bb38f4c93a8 100644 --- a/slobrok/src/vespa/slobrok/server/selfcheck.cpp +++ b/slobrok/src/vespa/slobrok/server/selfcheck.cpp @@ -1,10 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> - -#include <vespa/log/log.h> -LOG_SETUP(".selfcheck"); - -#include <vespa/fnet/frt/frt.h> #include "selfcheck.h" #include "ok_state.h" @@ -14,6 +8,9 @@ LOG_SETUP(".selfcheck"); #include "managed_rpc_server.h" #include "random.h" +#include <vespa/log/log.h> +LOG_SETUP(".selfcheck"); + namespace slobrok { diff --git a/slobrok/src/vespa/slobrok/server/slobrokserver.cpp b/slobrok/src/vespa/slobrok/server/slobrokserver.cpp index 731a2a1da60..3574caa2553 100644 --- a/slobrok/src/vespa/slobrok/server/slobrokserver.cpp +++ b/slobrok/src/vespa/slobrok/server/slobrokserver.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "slobrokserver.h" diff --git a/slobrok/src/vespa/slobrok/server/visible_map.h b/slobrok/src/vespa/slobrok/server/visible_map.h index 11702ddb92a..a6cebd787ce 100644 --- a/slobrok/src/vespa/slobrok/server/visible_map.h +++ b/slobrok/src/vespa/slobrok/server/visible_map.h @@ -1,14 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> -#include <vespa/vespalib/util/hashmap.h> -#include <vespa/vespalib/util/gencnt.h> -#include <vector> -#include <string> - #include "history.h" #include "named_service.h" +#include <vespa/vespalib/util/hashmap.h> namespace slobrok { diff --git a/socket_test/pom.xml b/socket_test/pom.xml index 12db1c30ab8..a681a82c52e 100644 --- a/socket_test/pom.xml +++ b/socket_test/pom.xml @@ -8,6 +8,11 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>socket_test</artifactId> <version>6-SNAPSHOT</version> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> <packaging>jar</packaging> <build> <plugins> diff --git a/staging_vespalib/src/tests/benchmark/testbase.cpp b/staging_vespalib/src/tests/benchmark/testbase.cpp index 93f5c9b975a..6de6f66f76b 100644 --- a/staging_vespalib/src/tests/benchmark/testbase.cpp +++ b/staging_vespalib/src/tests/benchmark/testbase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testbase.h" #include <vespa/fastos/timestamp.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".testbase"); diff --git a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp index 90fcdb8f94f..4af70f7e6fe 100644 --- a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp +++ b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/xmlserializable.h> +#include <vespa/vespalib/util/xmlstream.h> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp index 10bc2af51ce..54ac8363bf6 100644 --- a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp +++ b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp @@ -8,7 +8,8 @@ #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/util/exceptions.h> -#include <assert.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> namespace vespalib { @@ -129,9 +130,8 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen) continue; // Some illegal chars will be skipped. } else if (curchar == -1) { // Other illegal characters will generate failure - throw vespalib::IllegalArgumentException(vespalib::make_string( - "Illegal base64 character %u found.", - (unsigned int) *thischar), VESPA_STRLOC); + throw IllegalArgumentException(make_string("Illegal base64 character %u found.", + (unsigned int) *thischar), VESPA_STRLOC); } else { // Four bytes from input (eqals three bytes in output) @@ -162,4 +162,3 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen) } } // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp index 32b9caab687..2c67a3c48c9 100644 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp @@ -3,7 +3,7 @@ #include "http_server.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/host_name.h> -#include <algorithm> +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h index c0b12272797..ea9c8002d6e 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h +++ b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h @@ -5,8 +5,6 @@ * @brief Superclass for objects implementing clone() deep copy. */ -#include <memory> - namespace vespalib { class Cloneable { diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index ca440428e0a..3b36b863707 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -18,5 +18,6 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT shutdownguard.cpp timer.cpp xmlserializable.cpp + xmlstream.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/staging_vespalib/src/vespa/vespalib/util/clock.h index 4a5611bbe5a..e17ec962fc9 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.h +++ b/staging_vespalib/src/vespa/vespalib/util/clock.h @@ -33,7 +33,7 @@ public: Clock(double timePeriod=0.100); ~Clock(); - fastos::TimeStamp getTimeNS(void) const { + fastos::TimeStamp getTimeNS() const { if (!_running) { setTime(); } @@ -41,7 +41,7 @@ public: } fastos::TimeStamp getTimeNSAssumeRunning() const { return _timeNS; } - void stop(void); + void stop(); }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp index 7c581c89af0..576343245e4 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -2,6 +2,7 @@ #include "document_runnable.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> namespace document { diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 537b44c49eb..354d2e013ed 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/growablebytebuffer.h> +#include "growablebytebuffer.h" +#include <arpa/inet.h> using namespace vespalib; diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index fd65f0134bc..fdf090d22f9 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/stllike/string.h> diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp index e8f242879d8..c7a7ac34ef3 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp @@ -4,6 +4,7 @@ #include "jsonexception.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib { @@ -18,7 +19,7 @@ JsonStream::getStateName(const State& s) { throw IllegalStateException("Control should not reach this point", VESPA_STRLOC); } -JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents) +JsonStream::JsonStream(asciistream& as, bool createIndents) : _writer(as) { if (createIndents) _writer.setPretty(); @@ -28,7 +29,7 @@ JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents) JsonStream::~JsonStream() {} JsonStream& -JsonStream::operator<<(vespalib::stringref value) +JsonStream::operator<<(stringref value) { if (_state.empty()) { fail("Stream already finalized. Can't add a string value."); @@ -314,7 +315,7 @@ JsonStream::finalize() string JsonStream::getStateString() const { - vespalib::asciistream as; + asciistream as; for (auto it(_state.begin()), mt(_state.end()); it != mt; it++) { switch (it->state) { case State::OBJECT_EXPECTING_KEY: @@ -343,7 +344,7 @@ JsonStream::getStateString() const return as.str(); } -vespalib::string +string JsonStream::getJsonStreamState() const { asciistream report; @@ -360,4 +361,4 @@ JsonStream::fail(stringref error) const throw JsonStreamException(report.str(), "", VESPA_STRLOC); } -} // vespalib +} diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp index 8dd6995a6c1..d992b1b6db8 100644 --- a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/vespalib/util/librarypool.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib { @@ -17,14 +19,14 @@ LibraryPool::~LibraryPool() } void -LibraryPool::loadLibrary(const vespalib::stringref & libName) +LibraryPool::loadLibrary(const stringref & libName) { LockGuard guard(_lock); if (_libraries.find(libName) == _libraries.end()) { DynamicLibrarySP lib(new FastOS_DynamicLibrary); - vespalib::string file(libName); + string file(libName); if (!lib->Open(file.c_str())) { - vespalib::string error = lib->GetLastErrorString(); + string error = lib->GetLastErrorString(); throw IllegalArgumentException(make_string("Failed loading dynamic library '%s' due to '%s'.", file.c_str(), error.c_str())); } else { @@ -34,7 +36,7 @@ LibraryPool::loadLibrary(const vespalib::stringref & libName) } FastOS_DynamicLibrary * -LibraryPool::get(const vespalib::stringref & name) +LibraryPool::get(const stringref & name) { LockGuard guard(_lock); LibraryMap::const_iterator found(_libraries.find(name)); @@ -44,7 +46,7 @@ LibraryPool::get(const vespalib::stringref & name) } const FastOS_DynamicLibrary * -LibraryPool::get(const vespalib::stringref & name) const +LibraryPool::get(const stringref & name) const { LockGuard guard(_lock); LibraryMap::const_iterator found(_libraries.find(name)); diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h new file mode 100644 index 00000000000..6c4d8e37311 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +#include "polymorphicarraybase.h" + +namespace vespalib { + +/** + * Describes an interface an array of polymorphic types. + * The intention is to allow efficient implementations when that is possible + * while still enjoying the flexibility of the polymorph interface. + * It is not a full feldged Array implementation as std::vector. It contains just + * the minimum required to allow for efficient implementations for document::ArrayFieldValue. + * + * You specify the base type the interface shall provide. This base type must define + * virtual void assign(const B & rhs); + * For use with ComplexArrayT your type also need + * virtual T * clone() const; + */ +template<typename B> +class IArrayT : public IArrayBase { +public: + class iterator { + public: + iterator(IArrayT &a, size_t i) : _a(&a), _i(i) {} + iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } + iterator &operator++() { + ++_i; + return *this; + } + iterator operator++(int) { + iterator other(*this); + ++_i; + return other; + } + bool operator==(const iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const iterator &other) const { return (_i != other._i) || (_a != other._a); } + B &operator*() { return (*_a)[_i]; } + B *operator->() { return &(*_a)[_i]; } + friend ssize_t operator-(const iterator &a, const iterator &b) { return a._i - b._i; } + private: + IArrayT *_a; + size_t _i; + }; + + class const_iterator { + public: + const_iterator(const IArrayT &a, size_t i) : _a(&a), _i(i) {} + const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } + const_iterator &operator++() { + ++_i; + return *this; + } + const_iterator operator++(int) { + const_iterator other(*this); + ++_i; + return other; + } + bool operator==(const const_iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const const_iterator &other) const { return (_i != other._i) || (_a != other._a); } + const B &operator*() const { return (*_a)[_i]; } + const B *operator->() const { return &(*_a)[_i]; } + size_t operator-(const const_iterator &b) const { return _i - b._i; } + private: + const IArrayT *_a; + size_t _i; + }; + + typedef std::unique_ptr<IArrayT> UP; + virtual const B &operator[](size_t i) const = 0; + virtual B &operator[](size_t i) = 0; + virtual IArrayT *clone() const override = 0; + virtual iterator erase(iterator it) = 0; + virtual const_iterator begin() const { return const_iterator(*this, 0); } + virtual const_iterator end() const { return const_iterator(*this, size()); } + virtual iterator begin() { return iterator(*this, 0); } + virtual iterator end() { return iterator(*this, size()); } + virtual void push_back(const B &v) = 0; +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h new file mode 100644 index 00000000000..0fccd6cabb7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +namespace vespalib { + +class IArrayBase { +public: + virtual ~IArrayBase() {} + virtual void resize(size_t sz) = 0; + virtual void reserve(size_t sz) = 0; + virtual void clear() = 0; + virtual IArrayBase *clone() const = 0; + virtual size_t size() const = 0; + bool empty() const { return size() == 0; } +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h index 636586f839e..646ca522ba3 100644 --- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h @@ -2,75 +2,12 @@ // #pragma once +#include "polymorphicarray.h" #include <vespa/vespalib/util/memory.h> +#include <vector> namespace vespalib { -/** - * Describes an interface an array of polymorphic types. - * The intention is to allow efficient implementations when that is possible - * while still enjoying the flexibility of the polymorph interface. - * It is not a full feldged Array implementation as std::vector. It contains just - * the minimum required to allow for efficient implementations for document::ArrayFieldValue. - * - * You specify the base type the interface shall provide. This base type must define - * virtual void assign(const B & rhs); - * For use with ComplexArrayT your type also need - * virtual T * clone() const; - */ -template<typename B> -class IArrayT -{ -public: - class iterator { - public: - iterator(IArrayT & a, size_t i) : _a(&a), _i(i) { } - iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } - iterator& operator++() { ++_i; return *this; } - iterator operator++(int) { iterator other(*this); ++_i; return other; } - bool operator==(const iterator & other) const { return (_a == other._a) && (_i == other._i); } - bool operator!=(const iterator & other) const { return (_i != other._i) || (_a != other._a); } - B & operator*() { return (*_a)[_i]; } - B * operator->() { return &(*_a)[_i]; } - friend ssize_t operator - (const iterator & a, const iterator & b) { return a._i - b._i; } - private: - IArrayT * _a; - size_t _i; - }; - class const_iterator { - public: - const_iterator(const IArrayT & a, size_t i) : _a(&a), _i(i) { } - const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } - const_iterator& operator++() { ++_i; return *this; } - const_iterator operator++(int) { const_iterator other(*this); ++_i; return other; } - bool operator==(const const_iterator & other) const { return (_a == other._a) && (_i == other._i); } - bool operator!=(const const_iterator & other) const { return (_i != other._i) || (_a != other._a); } - const B & operator*() const { return (*_a)[_i]; } - const B * operator->() const { return &(*_a)[_i]; } - size_t operator - (const const_iterator & b) const { return _i - b._i; } - private: - const IArrayT * _a; - size_t _i; - }; - typedef std::unique_ptr<IArrayT> UP; - - virtual ~IArrayT() { } - virtual const B & operator [] (size_t i) const = 0; - virtual B & operator [] (size_t i) = 0; - virtual void resize(size_t sz) = 0; - virtual void reserve(size_t sz) = 0; - virtual void clear() = 0; - virtual IArrayT * clone() const = 0; - virtual size_t size() const = 0; - virtual iterator erase(iterator it) = 0; - virtual const_iterator begin() const { return const_iterator(*this, 0); } - virtual const_iterator end() const { return const_iterator(*this, size()); } - virtual iterator begin() { return iterator(*this, 0); } - virtual iterator end() { return iterator(*this, size()); } - bool empty() const { return size() == 0; } - virtual void push_back(const B & v) = 0; -}; - template <typename T, typename B> class PrimitiveArrayT : public IArrayT<B> { diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp index 41ff561011b..6573390c712 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/programoptions.h> - -#include <iostream> +#include "programoptions.h" +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/lexical_cast.hpp> diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp index 58a5489647d..357b3d94992 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp @@ -1,449 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/xmlserializable.hpp> -#include <vespa/vespalib/encoding/base64.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vector> -#include <cassert> +#include "xmlserializable.h" +#include "xmlstream.h" +#include <sstream> namespace vespalib { namespace xml { -namespace { - - std::vector<bool> getLegalIdentifierFirstCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; - for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; - vec[':'] = true; - vec['_'] = true; - return vec; - } - - std::vector<bool> getLegalIdentifierCharacters() { - std::vector<bool> vec(getLegalIdentifierFirstCharacters()); - vec['-'] = true; - vec['.'] = true; - for (uint32_t i='0'; i<='9'; ++i) { - vec[i] = true; - } - return vec; - } - - std::vector<bool> getBinaryCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i=0; i<32; ++i) { - vec[i] = true; - } - vec['\t'] = false; - vec['\n'] = false; - vec['\r'] = false; - vec['\f'] = false; - return vec; - } - - std::vector<bool> getEscapedXmlCharacters() { - std::vector<bool> vec(256, false); - for (uint32_t i=0; i<32; ++i) { - vec[i] = true; - } - vec['\n'] = false; - vec['<'] = true; - vec['>'] = true; - vec['&'] = true; - return vec; - } - - std::vector<bool> legalIdentifierFirstChar( - getLegalIdentifierFirstCharacters()); - std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); - std::vector<bool> binaryChars = getBinaryCharacters(); - std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); - - bool containsBinaryCharacters(const std::string& s) { - for (int i=0, n=s.size(); i<n; ++i) { - if (binaryChars[static_cast<uint8_t>(s[i])]) return true; - } - return false; - } - - const std::string xmlAttributeEscape(const std::string& s) { - vespalib::asciistream ost; - for (uint32_t i=0, n=s.size(); i<n; ++i) { - if (s[i] == '"' || s[i] == '\n' - || escapedXmlChars[static_cast<uint8_t>(s[i])]) - { - if (s[i] == '<') ost << "<"; - else if (s[i] == '>') ost << ">"; - else if (s[i] == '&') ost << "&"; - else if (s[i] == '"') ost << """; - else { - ost << "&#" << (int) s[i] << ";"; - } - } else { - ost << s[i]; - } - } - return ost.str(); - } - - void writeEscaped(std::ostream& out, const std::string& s) { - for (uint32_t i=0, n=s.size(); i<n; ++i) { - if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { - if (s[i] == '<') out << "<"; - else if (s[i] == '>') out << ">"; - else if (s[i] == '&') out << "&"; - else { - out << "&#" << (int) s[i] << ";"; - } - } else { - out << s[i]; - } - } - } - - void writeBase64Encoded(std::ostream& out, const std::string& s) { - out << vespalib::Base64::encode(&s[0], s.size()); - } -} - -bool isLegalName(const std::string& name) { - if (name.size() == 0) return false; - if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; - for (int i=1, n=name.size(); i<n; ++i) { - if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; - } - return true; -} - -void convertToLegalName(std::string& name) { - if (name.size() == 0) { - name == "__no_name__"; - } else { - if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { - name[0] = '_'; - } - for (int i=1, n=name.size(); i<n; ++i) { - if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { - name[i] = '_'; - } - } - } -} - -XmlOutputStream::XmlOutputStream(std::ostream& ostream, - const std::string& indent) - : _indent(indent), - _wrappedStream(ostream), - _tagStack(), - _cachedTag(), - _cachedAttributes(), - _cachedContent() -{ -} - -XmlAttribute::~XmlAttribute() -{ -} - -XmlContent::~XmlContent() -{ -} - -XmlOutputStream::~XmlOutputStream() -{ -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlTag& tag) -{ - //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " - // << (void*) _cachedTag.get() << "\n"; - if (_cachedTag.get() != 0) flush(false); - _cachedTag.reset(new XmlTag(tag)); - _cachedContentType = XmlContent::AUTO; - //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlAttribute& attribute) -{ - //std::cerr << "Adding attribute\n"; - if (_cachedTag.get() == 0) { - throw vespalib::IllegalStateException("Cannot add attribute " - + attribute.getName() + ", as no tag is open"); - } - _cachedAttributes.push_back(attribute); - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlEndTag&) -{ - //std::cerr << "Adding endtag\n"; - if (_cachedTag.get()) { - flush(true); - _cachedContentType = XmlContent::ESCAPED; - } else if (_tagStack.empty()) { - throw vespalib::IllegalStateException("No open tags left to end"); - } else { - for (uint32_t i=1; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << "</" << _tagStack.back() << ">"; - _tagStack.pop_back(); - if (!_tagStack.empty()) _wrappedStream << '\n'; - _cachedContentType = XmlContent::ESCAPED; - } - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlContent& content) -{ - //std::cerr << "Adding content\n"; - if (_cachedTag.get() == 0 && _tagStack.empty()) { - throw vespalib::IllegalStateException( - "No open tag to write content in"); - } - if (_cachedTag.get() != 0) { - //std::cerr << "Content is '" << content.getContent() << "'\n"; - if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok - } else if (_cachedContentType == XmlContent::AUTO) { - _cachedContentType = content.getType(); - } else if (_cachedContentType != content.getType()) { - throw vespalib::IllegalStateException( - "Have already added content of different type"); - } - _cachedContent.push_back(content); - } else { - if (content.getType() == XmlContent::BASE64) { - throw vespalib::IllegalStateException( - "Cannot add Base64 encoded content after tag content"); - } - for (uint32_t i=0; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << content.getContent() << '\n'; - } - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const XmlSerializable& serializable) -{ - //std::cerr << "Adding serializable\n"; - serializable.printXml(*this); - return *this; -} - -XmlOutputStream& -XmlOutputStream::operator<<(const std::string& content) -{ - //std::cerr << "Adding content string\n"; - return *this << XmlContent(content); -} - -XmlOutputStream& -XmlOutputStream::operator<<(char c) -{ - return *this << XmlContent(std::string(&c, 1)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(int32_t i) -{ - return *this << XmlContent(vespalib::make_string("%d", i)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(int64_t i) -{ - return *this << XmlContent(vespalib::make_string("%" PRId64, i)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(float f) -{ - return *this << XmlContent(vespalib::make_string("%g", f)); -} - -XmlOutputStream& -XmlOutputStream::operator<<(double d) -{ - return *this << XmlContent(vespalib::make_string("%g", d)); -} - -void -XmlOutputStream::flush(bool endTag) -{ - //std::cerr << "Flushing\n"; - if (_cachedTag.get() == 0) { - throw vespalib::IllegalStateException("Cannot write non-existing tag"); - } - for (uint32_t i=0; i<_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - _wrappedStream << '<' << _cachedTag->getName(); - for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); - it != _cachedAttributes.end(); ++it) - { - _wrappedStream << ' ' << it->getName() << "=\"" - << xmlAttributeEscape(it->getValue()) << '"'; - } - _cachedAttributes.clear(); - if (_cachedContent.empty() && endTag) { - _wrappedStream << "/>\n"; - } else if (_cachedContent.empty()) { - _wrappedStream << ">\n"; - _tagStack.push_back(_cachedTag->getName()); - } else { - if (_cachedContentType == XmlContent::AUTO) { - _cachedContentType = XmlContent::ESCAPED; - for (std::list<XmlContent>::const_iterator it - = _cachedContent.begin(); it != _cachedContent.end(); ++it) - { - if (containsBinaryCharacters(it->getContent())) { - _cachedContentType = XmlContent::BASE64; - break; - } - } - } - if (_cachedContentType == XmlContent::BASE64) { - _wrappedStream << " binaryencoding=\"base64\""; - } - _wrappedStream << '>'; - for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); - it != _cachedContent.end(); ++it) - { - if (!endTag) { - _wrappedStream << '\n'; - for (uint32_t i=0; i<=_tagStack.size(); ++i) { - _wrappedStream << _indent; - } - } - switch (_cachedContentType) { - case XmlContent::ESCAPED: { - writeEscaped(_wrappedStream, it->getContent()); - break; - } - case XmlContent::BASE64: { - writeBase64Encoded(_wrappedStream, it->getContent()); - break; - } - default: assert(false); - } - } - _cachedContent.clear(); - if (endTag) { - _wrappedStream << "</" << _cachedTag->getName() << ">\n"; - } else { - _wrappedStream << '\n'; - _tagStack.push_back(_cachedTag->getName()); - } - } - _cachedTag.reset(0); -} - -XmlTag::XmlTag(const XmlTag& tag) - : _name(tag._name), - _attributes(), - _content(), - _flags(tag._flags) -{ -} - -XmlTag::~XmlTag() {} - -XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) - : _name(name), - _attributes(), - _content(), - _flags(flags) -{ - if (_flags == CONVERT_ILLEGAL_CHARACTERS) { - convertToLegalName(_name); - } - if (!isLegalName(_name)) { - throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " - "illegal XML characters and cannot be used as tag name"); - } -} - -XmlAttribute::XmlAttribute(const XmlAttribute& attribute) - : _name(attribute._name), - _value(attribute._value), - _next() -{ -} - -XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags) - : _name(name), - _value(), - _next() -{ - vespalib::asciistream ost; - if (flags & HEX) ost << vespalib::hex << "0x"; - ost << value; - _value = ost.str(); - if (!isLegalName(name)) { - throw vespalib::IllegalArgumentException("Name '" + name + "' contains " - "illegal XML characters and cannot be used as attribute name"); - } -} - -XmlEndTag::XmlEndTag() -{ -} - -XmlContent::XmlContent(Type type) - : _type(type), - _content(), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent() - : _type(AUTO), - _content(), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent(const XmlContent& content) - : _type(content._type), - _content(content._content), - _nextContent(), - _nextTag() -{ -} - -XmlContent::XmlContent(const std::string& value) - : _type(AUTO), - _content(value), - _nextContent(), - _nextTag() -{ -} - -XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) - : XmlContent(wrapper) -{ -} - -XmlContentWrapper::XmlContentWrapper(const char* value) - : XmlContent(std::string(value)) -{ -} - -XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) - : XmlContent(std::string(value, size)) -{ -} - std::string XmlSerializable::toXml(const std::string& indent) const { @@ -452,22 +15,6 @@ XmlSerializable::toXml(const std::string& indent) const printXml(xos); return ost.str(); } -using CharP = char *; -using ConstCharP = const char *; - -template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int); -template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int); } // xml } // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h index b688c699d76..a4ddb12ed8d 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h @@ -1,166 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file xmlserializable.h - * @ingroup util - * - * @brief Interfaces to be used for XML serialization. - * - * This file contains XML utility classes, to make XML serialization simple. - * Rather than users writing their own XML, these tools let you define a tree - * structure, and this library builds the XML for you. This ensures that you - * write legal XML and that stuff that needs to be escaped is. - * <p> - * It defines a superclass for XML serializable classes, called XmlSerializable. - * This is what classes that should be XML serializable will inherit. - * <p> - * When implementing the printXml() function in XmlSerializable, one will - * use the various XML helper classes defined here to build a tree structure - * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. - * Some subclasses exist of XmlContent to facilitate various types of content. - * <p> - * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it - * and it is responsible for writing all the XML code. This way, the XML - * serialization is done without interfering with regular output operators. - * <p> - * For example usage, refer to the unit test: - * vespalib/tests/xmlserializable/xmlserializabletest.cpp - * - */ #pragma once -#include <iosfwd> -#include <list> -#include <memory> +#include <string> namespace vespalib { namespace xml { -class XmlAttribute; -class XmlContent; class XmlOutputStream; -bool isLegalName(const std::string& name); - -enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; - -/** - * @class document::XmlTag - * - * @brief Start a new tag with given name. - */ -class XmlTag { - std::string _name; - std::unique_ptr<XmlAttribute> _attributes; - std::unique_ptr<XmlContent> _content; - XmlTagFlags _flags; -public: - XmlTag(const XmlTag&); - XmlTag(const std::string& name, XmlTagFlags = NONE); - ~XmlTag(); - - const std::string& getName() const { return _name; } -}; - -/** - * @class document::XmlEndTag - * - * @brief Indicates that current tag is closed. - */ -class XmlEndTag { -public: - XmlEndTag(); -}; - -/** - * @class document::XmlAttribute - * - * @brief Defined a single attribute within an XML tag. - * - * When adding an XML to an XML stream, the attribute will be added to the last - * tag added. This can not be called after the last tag opened in the stream is - * closed, so add all attributes before starting to add new XML child tags. - */ -class XmlAttribute { - std::string _name; - std::string _value; - std::unique_ptr<XmlAttribute> _next; -public: - enum Flag { NONE = 0x0, HEX = 0x1 }; - XmlAttribute(const XmlAttribute&); - /** Add any value that can be written to an ostringstream. */ - template<typename T> - XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE); - XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE); - ~XmlAttribute(); - - const std::string& getName() const { return _name; } - const std::string& getValue() const { return _value; } -}; - - -/** - * @class document::XmlContent - * - * XML content to be written to stream. By default it will autodetect whether to - * escape or base64 encode content. XmlOutputStream functions taking primitives - * will generate XmlContent instances. - */ -class XmlContent { -public: - enum Type { AUTO, ESCAPED, BASE64 }; -protected: - XmlContent(Type type); -private: - Type _type; - std::string _content; - std::unique_ptr<XmlContent> _nextContent; - std::unique_ptr<XmlTag> _nextTag; - -public: - XmlContent(); - XmlContent(const XmlContent&); - XmlContent(const std::string& value); - ~XmlContent(); - - Type getType() const { return _type; } - const std::string& getContent() const { return _content; } -}; - -/** - * @class document::XmlEscapedContent - * - * Token used to tell that this content field should only be XML escaped. - */ -class XmlEscapedContent : public XmlContent { -public: - XmlEscapedContent() : XmlContent(ESCAPED) {} -}; - -/** - * @class document::XmlBase64Content - * - * Token used to tell that this content field should always be base64 encoded. - */ -class XmlBase64Content : public XmlContent { -public: - XmlBase64Content() : XmlContent(BASE64) {} -}; - -/** - * @class document::XmlContentWrapper - * - * A wrapper class for content that one doesn't want to copy or release - * ownership of. This wrapper merely takes pointer to data, and assumes it - * will stay alive as long as needed. - */ -class XmlContentWrapper : public XmlContent { -public: - XmlContentWrapper(const XmlContentWrapper&); - XmlContentWrapper(const char* value); - XmlContentWrapper(const char* value, uint32_t size); -}; - /** * @class document::XmlSerializable * @@ -178,51 +26,6 @@ public: virtual std::string toXml(const std::string& indent = "") const; }; -/** - * @class document::XmlOutputStream - * - * @brief std::ostream wrapper, only accepting data that will become XML. - * - * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have - * been written. Call isFinalized() to ensure that you have closed all the tags - * that have been opened. Within a tag, the stream will cache some information, - * as more information might be required before knowing what to print. - */ -class XmlOutputStream { - const std::string _indent; - std::ostream& _wrappedStream; - std::list<std::string> _tagStack; - std::unique_ptr<XmlTag> _cachedTag; - std::list<XmlAttribute> _cachedAttributes; - std::list<XmlContent> _cachedContent; - XmlContent::Type _cachedContentType; - - void flush(bool endTag); - -public: - - XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); - ~XmlOutputStream(); - - bool isFinalized() const - { return (_tagStack.empty() && _cachedTag.get() == 0); } - - std::ostream& getWrappedStream() { return _wrappedStream; } - - XmlOutputStream& operator<<(const XmlTag& tag); - XmlOutputStream& operator<<(const XmlAttribute& attribute); - XmlOutputStream& operator<<(const XmlEndTag& endtag); - XmlOutputStream& operator<<(const XmlContent& content); - XmlOutputStream& operator<<(const XmlSerializable& serializable); - - XmlOutputStream& operator<<(const std::string& content); - XmlOutputStream& operator<<(char c); - XmlOutputStream& operator<<(int32_t i); - XmlOutputStream& operator<<(int64_t i); - XmlOutputStream& operator<<(float f); - XmlOutputStream& operator<<(double d); -}; - } // xml // The XmlSerializable and XmlOutputStream is often used in header files @@ -230,8 +33,8 @@ public: // vespalib namespace with all the other classes, use // "using namespace vespalib::xml" within your printXml functions -typedef vespalib::xml::XmlOutputStream XmlOutputStream; -typedef vespalib::xml::XmlSerializable XmlSerializable; +using XmlSerializable = vespalib::xml::XmlSerializable; +using XmlOutputStream = vespalib::xml::XmlOutputStream; } // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp new file mode 100644 index 00000000000..16fce61ddd1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp @@ -0,0 +1,463 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "xmlstream.hpp" +#include <vespa/vespalib/encoding/base64.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +namespace vespalib::xml { + +namespace { + + std::vector<bool> getLegalIdentifierFirstCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; + for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; + vec[':'] = true; + vec['_'] = true; + return vec; + } + + std::vector<bool> getLegalIdentifierCharacters() { + std::vector<bool> vec(getLegalIdentifierFirstCharacters()); + vec['-'] = true; + vec['.'] = true; + for (uint32_t i='0'; i<='9'; ++i) { + vec[i] = true; + } + return vec; + } + + std::vector<bool> getBinaryCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\t'] = false; + vec['\n'] = false; + vec['\r'] = false; + vec['\f'] = false; + return vec; + } + + std::vector<bool> getEscapedXmlCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\n'] = false; + vec['<'] = true; + vec['>'] = true; + vec['&'] = true; + return vec; + } + + std::vector<bool> legalIdentifierFirstChar( + getLegalIdentifierFirstCharacters()); + std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); + std::vector<bool> binaryChars = getBinaryCharacters(); + std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); + + bool containsBinaryCharacters(const std::string& s) { + for (int i=0, n=s.size(); i<n; ++i) { + if (binaryChars[static_cast<uint8_t>(s[i])]) return true; + } + return false; + } + + const std::string xmlAttributeEscape(const std::string& s) { + vespalib::asciistream ost; + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (s[i] == '"' || s[i] == '\n' + || escapedXmlChars[static_cast<uint8_t>(s[i])]) + { + if (s[i] == '<') ost << "<"; + else if (s[i] == '>') ost << ">"; + else if (s[i] == '&') ost << "&"; + else if (s[i] == '"') ost << """; + else { + ost << "&#" << (int) s[i] << ";"; + } + } else { + ost << s[i]; + } + } + return ost.str(); + } + + void writeEscaped(std::ostream& out, const std::string& s) { + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { + if (s[i] == '<') out << "<"; + else if (s[i] == '>') out << ">"; + else if (s[i] == '&') out << "&"; + else { + out << "&#" << (int) s[i] << ";"; + } + } else { + out << s[i]; + } + } + } + + void writeBase64Encoded(std::ostream& out, const std::string& s) { + out << vespalib::Base64::encode(&s[0], s.size()); + } +} + +bool isLegalName(const std::string& name) { + if (name.size() == 0) return false; + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; + } + return true; +} + +void convertToLegalName(std::string& name) { + if (name.size() == 0) { + name == "__no_name__"; + } else { + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { + name[0] = '_'; + } + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { + name[i] = '_'; + } + } + } +} + +XmlOutputStream::XmlOutputStream(std::ostream& ostream, + const std::string& indent) + : _indent(indent), + _wrappedStream(ostream), + _tagStack(), + _cachedTag(), + _cachedAttributes(), + _cachedContent() +{ +} + +XmlAttribute::~XmlAttribute() +{ +} + +XmlContent::~XmlContent() +{ +} + +XmlOutputStream::~XmlOutputStream() +{ +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlTag& tag) +{ + //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " + // << (void*) _cachedTag.get() << "\n"; + if (_cachedTag.get() != 0) flush(false); + _cachedTag.reset(new XmlTag(tag)); + _cachedContentType = XmlContent::AUTO; + //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlAttribute& attribute) +{ + //std::cerr << "Adding attribute\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot add attribute " + + attribute.getName() + ", as no tag is open"); + } + _cachedAttributes.push_back(attribute); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlEndTag&) +{ + //std::cerr << "Adding endtag\n"; + if (_cachedTag.get()) { + flush(true); + _cachedContentType = XmlContent::ESCAPED; + } else if (_tagStack.empty()) { + throw vespalib::IllegalStateException("No open tags left to end"); + } else { + for (uint32_t i=1; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << "</" << _tagStack.back() << ">"; + _tagStack.pop_back(); + if (!_tagStack.empty()) _wrappedStream << '\n'; + _cachedContentType = XmlContent::ESCAPED; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlContent& content) +{ + //std::cerr << "Adding content\n"; + if (_cachedTag.get() == 0 && _tagStack.empty()) { + throw vespalib::IllegalStateException( + "No open tag to write content in"); + } + if (_cachedTag.get() != 0) { + //std::cerr << "Content is '" << content.getContent() << "'\n"; + if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok + } else if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = content.getType(); + } else if (_cachedContentType != content.getType()) { + throw vespalib::IllegalStateException( + "Have already added content of different type"); + } + _cachedContent.push_back(content); + } else { + if (content.getType() == XmlContent::BASE64) { + throw vespalib::IllegalStateException( + "Cannot add Base64 encoded content after tag content"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << content.getContent() << '\n'; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlSerializable& serializable) +{ + //std::cerr << "Adding serializable\n"; + serializable.printXml(*this); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const std::string& content) +{ + //std::cerr << "Adding content string\n"; + return *this << XmlContent(content); +} + +XmlOutputStream& +XmlOutputStream::operator<<(char c) +{ + return *this << XmlContent(std::string(&c, 1)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int32_t i) +{ + return *this << XmlContent(vespalib::make_string("%d", i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int64_t i) +{ + return *this << XmlContent(vespalib::make_string("%" PRId64, i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(float f) +{ + return *this << XmlContent(vespalib::make_string("%g", f)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(double d) +{ + return *this << XmlContent(vespalib::make_string("%g", d)); +} + +void +XmlOutputStream::flush(bool endTag) +{ + //std::cerr << "Flushing\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot write non-existing tag"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << '<' << _cachedTag->getName(); + for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); + it != _cachedAttributes.end(); ++it) + { + _wrappedStream << ' ' << it->getName() << "=\"" + << xmlAttributeEscape(it->getValue()) << '"'; + } + _cachedAttributes.clear(); + if (_cachedContent.empty() && endTag) { + _wrappedStream << "/>\n"; + } else if (_cachedContent.empty()) { + _wrappedStream << ">\n"; + _tagStack.push_back(_cachedTag->getName()); + } else { + if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = XmlContent::ESCAPED; + for (std::list<XmlContent>::const_iterator it + = _cachedContent.begin(); it != _cachedContent.end(); ++it) + { + if (containsBinaryCharacters(it->getContent())) { + _cachedContentType = XmlContent::BASE64; + break; + } + } + } + if (_cachedContentType == XmlContent::BASE64) { + _wrappedStream << " binaryencoding=\"base64\""; + } + _wrappedStream << '>'; + for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); + it != _cachedContent.end(); ++it) + { + if (!endTag) { + _wrappedStream << '\n'; + for (uint32_t i=0; i<=_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + } + switch (_cachedContentType) { + case XmlContent::ESCAPED: { + writeEscaped(_wrappedStream, it->getContent()); + break; + } + case XmlContent::BASE64: { + writeBase64Encoded(_wrappedStream, it->getContent()); + break; + } + default: assert(false); + } + } + _cachedContent.clear(); + if (endTag) { + _wrappedStream << "</" << _cachedTag->getName() << ">\n"; + } else { + _wrappedStream << '\n'; + _tagStack.push_back(_cachedTag->getName()); + } + } + _cachedTag.reset(0); +} + +XmlTag::XmlTag(const XmlTag& tag) + : _name(tag._name), + _attributes(), + _content(), + _flags(tag._flags) +{ +} + +XmlTag::~XmlTag() {} + +XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) + : _name(name), + _attributes(), + _content(), + _flags(flags) +{ + if (_flags == CONVERT_ILLEGAL_CHARACTERS) { + convertToLegalName(_name); + } + if (!isLegalName(_name)) { + throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " + "illegal XML characters and cannot be used as tag name"); + } +} + +XmlAttribute::XmlAttribute(const XmlAttribute& attribute) + : _name(attribute._name), + _value(attribute._value), + _next() +{ +} + +XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags) + : _name(name), + _value(), + _next() +{ + vespalib::asciistream ost; + if (flags & HEX) ost << vespalib::hex << "0x"; + ost << value; + _value = ost.str(); + if (!isLegalName(name)) { + throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + "illegal XML characters and cannot be used as attribute name"); + } +} + +XmlEndTag::XmlEndTag() +{ +} + +XmlContent::XmlContent(Type type) + : _type(type), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent() + : _type(AUTO), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const XmlContent& content) + : _type(content._type), + _content(content._content), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const std::string& value) + : _type(AUTO), + _content(value), + _nextContent(), + _nextTag() +{ +} + +XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) + : XmlContent(wrapper) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value) + : XmlContent(std::string(value)) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) + : XmlContent(std::string(value, size)) +{ +} + +using CharP = char *; +using ConstCharP = const char *; + +template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int); + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.h b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h new file mode 100644 index 00000000000..8ed781e57c3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h @@ -0,0 +1,210 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file xmlserializable.h + * @ingroup util + * + * @brief Interfaces to be used for XML serialization. + * + * This file contains XML utility classes, to make XML serialization simple. + * Rather than users writing their own XML, these tools let you define a tree + * structure, and this library builds the XML for you. This ensures that you + * write legal XML and that stuff that needs to be escaped is. + * <p> + * It defines a superclass for XML serializable classes, called XmlSerializable. + * This is what classes that should be XML serializable will inherit. + * <p> + * When implementing the printXml() function in XmlSerializable, one will + * use the various XML helper classes defined here to build a tree structure + * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. + * Some subclasses exist of XmlContent to facilitate various types of content. + * <p> + * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it + * and it is responsible for writing all the XML code. This way, the XML + * serialization is done without interfering with regular output operators. + * <p> + * For example usage, refer to the unit test: + * vespalib/tests/xmlserializable/xmlserializabletest.cpp + * + */ + +#pragma once + +#include "xmlserializable.h" +#include <iosfwd> +#include <list> +#include <memory> + +namespace vespalib::xml { + +class XmlAttribute; +class XmlContent; +class XmlOutputStream; + +bool isLegalName(const std::string& name); + +enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; + +/** + * @class document::XmlTag + * + * @brief Start a new tag with given name. + */ +class XmlTag { + std::string _name; + std::unique_ptr<XmlAttribute> _attributes; + std::unique_ptr<XmlContent> _content; + XmlTagFlags _flags; +public: + XmlTag(const XmlTag&); + XmlTag(const std::string& name, XmlTagFlags = NONE); + ~XmlTag(); + + const std::string& getName() const { return _name; } +}; + +/** + * @class document::XmlEndTag + * + * @brief Indicates that current tag is closed. + */ +class XmlEndTag { +public: + XmlEndTag(); +}; + +/** + * @class document::XmlAttribute + * + * @brief Defined a single attribute within an XML tag. + * + * When adding an XML to an XML stream, the attribute will be added to the last + * tag added. This can not be called after the last tag opened in the stream is + * closed, so add all attributes before starting to add new XML child tags. + */ +class XmlAttribute { + std::string _name; + std::string _value; + std::unique_ptr<XmlAttribute> _next; +public: + enum Flag { NONE = 0x0, HEX = 0x1 }; + XmlAttribute(const XmlAttribute&); + /** Add any value that can be written to an ostringstream. */ + template<typename T> + XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE); + XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE); + ~XmlAttribute(); + + const std::string& getName() const { return _name; } + const std::string& getValue() const { return _value; } +}; + + +/** + * @class document::XmlContent + * + * XML content to be written to stream. By default it will autodetect whether to + * escape or base64 encode content. XmlOutputStream functions taking primitives + * will generate XmlContent instances. + */ +class XmlContent { +public: + enum Type { AUTO, ESCAPED, BASE64 }; +protected: + XmlContent(Type type); +private: + Type _type; + std::string _content; + std::unique_ptr<XmlContent> _nextContent; + std::unique_ptr<XmlTag> _nextTag; + +public: + XmlContent(); + XmlContent(const XmlContent&); + XmlContent(const std::string& value); + ~XmlContent(); + + Type getType() const { return _type; } + const std::string& getContent() const { return _content; } +}; + +/** + * @class document::XmlEscapedContent + * + * Token used to tell that this content field should only be XML escaped. + */ +class XmlEscapedContent : public XmlContent { +public: + XmlEscapedContent() : XmlContent(ESCAPED) {} +}; + +/** + * @class document::XmlBase64Content + * + * Token used to tell that this content field should always be base64 encoded. + */ +class XmlBase64Content : public XmlContent { +public: + XmlBase64Content() : XmlContent(BASE64) {} +}; + +/** + * @class document::XmlContentWrapper + * + * A wrapper class for content that one doesn't want to copy or release + * ownership of. This wrapper merely takes pointer to data, and assumes it + * will stay alive as long as needed. + */ +class XmlContentWrapper : public XmlContent { +public: + XmlContentWrapper(const XmlContentWrapper&); + XmlContentWrapper(const char* value); + XmlContentWrapper(const char* value, uint32_t size); +}; + +/** + * @class document::XmlOutputStream + * + * @brief std::ostream wrapper, only accepting data that will become XML. + * + * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have + * been written. Call isFinalized() to ensure that you have closed all the tags + * that have been opened. Within a tag, the stream will cache some information, + * as more information might be required before knowing what to print. + */ +class XmlOutputStream { + const std::string _indent; + std::ostream& _wrappedStream; + std::list<std::string> _tagStack; + std::unique_ptr<XmlTag> _cachedTag; + std::list<XmlAttribute> _cachedAttributes; + std::list<XmlContent> _cachedContent; + XmlContent::Type _cachedContentType; + + void flush(bool endTag); + +public: + + XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); + ~XmlOutputStream(); + + bool isFinalized() const + { return (_tagStack.empty() && _cachedTag.get() == 0); } + + std::ostream& getWrappedStream() { return _wrappedStream; } + + XmlOutputStream& operator<<(const XmlTag& tag); + XmlOutputStream& operator<<(const XmlAttribute& attribute); + XmlOutputStream& operator<<(const XmlEndTag& endtag); + XmlOutputStream& operator<<(const XmlContent& content); + XmlOutputStream& operator<<(const XmlSerializable& serializable); + + XmlOutputStream& operator<<(const std::string& content); + XmlOutputStream& operator<<(char c); + XmlOutputStream& operator<<(int32_t i); + XmlOutputStream& operator<<(int64_t i); + XmlOutputStream& operator<<(float f); + XmlOutputStream& operator<<(double d); +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp index 7fcc964d8f1..243efe9e94b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp @@ -2,12 +2,11 @@ #pragma once -#include "xmlserializable.h" -#include <sstream> +#include "xmlstream.h" #include <vespa/vespalib/util/exceptions.h> +#include <sstream> -namespace vespalib { -namespace xml { +namespace vespalib::xml { template<typename T> XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t flags) @@ -20,10 +19,9 @@ XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t fla ost << value; _value = ost.str(); if (!isLegalName(name)) { - throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + throw IllegalArgumentException("Name '" + name + "' contains " "illegal XML characters and cannot be used as attribute name"); } } } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index fc57353c194..c8eb8666bed 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -1,36 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.standalone -import com.google.inject.{Key, AbstractModule, Injector, Inject} -import com.yahoo.config.application.api.{RuleConfigDeriver, FileRegistry, ApplicationPackage} +import java.io.{File, IOException} +import java.lang.{Boolean => JBoolean} +import java.nio.file.{FileSystems, Files, Path, Paths} + +import com.google.inject.name.Names +import com.google.inject.{AbstractModule, Inject, Injector, Key} +import com.yahoo.collections.CollectionUtil.first +import com.yahoo.config.application.api.{ApplicationPackage, FileRegistry, RuleConfigDeriver} +import com.yahoo.config.model.application.provider._ +import com.yahoo.config.model.builder.xml.XmlHelper +import com.yahoo.config.model.deploy.DeployState +import com.yahoo.config.model.{ApplicationConfigProducerRoot, ConfigModelRepo} import com.yahoo.config.provision.Zone -import com.yahoo.jdisc.application.Application +import com.yahoo.container.di.config.SubscriberFactory import com.yahoo.container.jdisc.ConfiguredApplication -import java.io.{IOException, File} -import com.yahoo.config.model.test.MockRoot -import com.yahoo.config.model.application.provider._ +import com.yahoo.container.standalone.Environment._ +import com.yahoo.container.standalone.StandaloneContainerApplication._ +import com.yahoo.io.IOUtils +import com.yahoo.jdisc.application.Application +import com.yahoo.text.XML import com.yahoo.vespa.defaults.Defaults import com.yahoo.vespa.model.VespaModel -import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder} -import org.w3c.dom.Element -import com.yahoo.config.model.builder.xml.XmlHelper -import com.yahoo.vespa.model.container.Container -import com.yahoo.collections.CollectionUtil.first import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder -import com.yahoo.io.IOUtils -import com.yahoo.container.di.config.SubscriberFactory -import StandaloneContainerApplication._ -import com.google.inject.name.Names -import scala.util.Try -import java.nio.file.{FileSystems, Path, Paths, Files} -import com.yahoo.config.model.{ConfigModelRepo, ApplicationConfigProducerRoot} -import scala.collection.JavaConversions._ -import com.yahoo.text.XML +import com.yahoo.vespa.model.container.Container import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking +import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ContainerModelBuilder} +import org.w3c.dom.Element -import java.lang.{ Boolean => JBoolean } -import Environment._ -import com.yahoo.config.model.deploy.DeployState +import scala.collection.JavaConversions._ +import scala.util.Try /** * @author tonytv @@ -99,7 +99,6 @@ object StandaloneContainerApplication { val packageName = "standalone_jdisc_container" val applicationLocationYinstVariable = s"$packageName.app_location" val deploymentProfileYinstVariable = s"$packageName.deployment_profile" - val manhattanHttpPortYinstVariable = s"$packageName.manhattan_http_port" val applicationPathName = Names.named(applicationLocationYinstVariable) @@ -146,21 +145,11 @@ object StandaloneContainerApplication { def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { optionalYinstVariable(deploymentProfileYinstVariable) match { case None => new ContainerModelBuilder(true, networkingOption) - case Some("manhattan") => new ManhattanContainerModelBuilder(manhattanHttpPort) case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables) case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") } } - def manhattanHttpPort: Int = { - val port = yinstVariable(manhattanHttpPortYinstVariable) - Try { - Integer.parseInt(port) - } filter( _ > 0) getOrElse { - throw new RuntimeException(s"$manhattanHttpPortYinstVariable is not a valid port: '$port'") - } - } - def createContainerModel(applicationPath: Path, fileRegistry: FileRegistry, preprocessedApplicationDir: File, diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala index 2705322ab32..00d84d36165 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala @@ -2,13 +2,11 @@ package com.yahoo.container.standalone +import com.yahoo.container.standalone.StandaloneContainerTest._ +import com.yahoo.vespa.model.AbstractService import org.junit.Assert._ import org.junit.Test -import org.hamcrest.CoreMatchers.is -import org.hamcrest.text.StringContainsInOrder.stringContainsInOrder -import StandaloneContainerTest._ -import com.yahoo.vespa.model.AbstractService -import java.util.Arrays.asList + import scala.util.Try @@ -79,29 +77,6 @@ class StandaloneContainerTest { } } - @Test - def manhattan_http_port_yinst_variable_must_be_set() { - System.clearProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable) - try { - StandaloneContainerApplication.manhattanHttpPort - fail("Port should be a required setting") - } catch { - case e: Exception => - assertThat(e.getMessage, stringContainsInOrder(asList("Environment variable not set", "manhattan_http_port"))) - } - } - - @Test - def manhattan_http_port_must_be_positive_integer() { - assertThat(setAndGetPort(1234), is(1234)) - assertTrue(Try { setAndGetPort(-1) }.isFailure) - assertTrue(Try { setAndGetPort( 0) }.isFailure) - } - - def setAndGetPort(port: Int): Int = { - System.setProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable, Integer.toString(port)) - StandaloneContainerApplication.manhattanHttpPort - } } object StandaloneContainerTest { diff --git a/statistics/.gitignore b/statistics/.gitignore index 85d32008813..324d6034fd1 100644 --- a/statistics/.gitignore +++ b/statistics/.gitignore @@ -6,5 +6,4 @@ target .settings .classpath .project - /pom.xml.build diff --git a/statistics/src/main/java/com/yahoo/.gitignore b/statistics/src/main/java/com/yahoo/.gitignore deleted file mode 100644 index 04204c7c9d0..00000000000 --- a/statistics/src/main/java/com/yahoo/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config diff --git a/statistics/src/main/java/com/yahoo/statistics/.gitignore b/statistics/src/main/java/com/yahoo/statistics/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/.gitignore +++ /dev/null diff --git a/storage/OWNERS b/storage/OWNERS index 11a58a546b1..4cef58fa4f9 100644 --- a/storage/OWNERS +++ b/storage/OWNERS @@ -1,2 +1,2 @@ vekterli -dybis + diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp index a625f6303ea..5fe5d9b58dc 100644 --- a/storage/src/tests/bucketdb/bucketinfotest.cpp +++ b/storage/src/tests/bucketdb/bucketinfotest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <boost/assign.hpp> #include <boost/random.hpp> diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index f9ae4f57fa5..227dd40b34d 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -16,6 +16,7 @@ #include <tests/common/testhelper.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/config/helper/configgetter.hpp> #include <future> diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp index 4a6157fc93e..1dd2576a264 100644 --- a/storage/src/tests/bucketdb/judyarraytest.cpp +++ b/storage/src/tests/bucketdb/judyarraytest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/judyarray.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <boost/assign.hpp> diff --git a/storage/src/tests/bucketdb/judymultimaptest.cpp b/storage/src/tests/bucketdb/judymultimaptest.cpp index 998bc25d0bf..ba662133e00 100644 --- a/storage/src/tests/bucketdb/judymultimaptest.cpp +++ b/storage/src/tests/bucketdb/judymultimaptest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/judymultimap.h> #include <vespa/storage/bucketdb/judymultimap.hpp> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/bucketmover/htmltabletest.cpp b/storage/src/tests/bucketmover/htmltabletest.cpp index 98cf68d489a..0d61cce3e55 100644 --- a/storage/src/tests/bucketmover/htmltabletest.cpp +++ b/storage/src/tests/bucketmover/htmltabletest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketmover/htmltable.h> #include <tests/common/testhelper.h> diff --git a/storage/src/tests/common/hostreporter/cpureportertest.cpp b/storage/src/tests/common/hostreporter/cpureportertest.cpp index 56a929c3aff..fbc5aa5e36d 100644 --- a/storage/src/tests/common/hostreporter/cpureportertest.cpp +++ b/storage/src/tests/common/hostreporter/cpureportertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/common/hostreporter/cpureporter.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/common/hostreporter/diskreportertest.cpp b/storage/src/tests/common/hostreporter/diskreportertest.cpp index 158a77c2e7e..fe14519dfe1 100644 --- a/storage/src/tests/common/hostreporter/diskreportertest.cpp +++ b/storage/src/tests/common/hostreporter/diskreportertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/common/hostreporter/diskreporter.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/common/hostreporter/memreportertest.cpp b/storage/src/tests/common/hostreporter/memreportertest.cpp index 3eedfd48a3c..741e7df917a 100644 --- a/storage/src/tests/common/hostreporter/memreportertest.cpp +++ b/storage/src/tests/common/hostreporter/memreportertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/common/hostreporter/memreporter.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/common/hostreporter/networkreportertest.cpp b/storage/src/tests/common/hostreporter/networkreportertest.cpp index cba5717adce..c8db438b627 100644 --- a/storage/src/tests/common/hostreporter/networkreportertest.cpp +++ b/storage/src/tests/common/hostreporter/networkreportertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/common/hostreporter/networkreporter.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/common/hostreporter/versionreportertest.cpp b/storage/src/tests/common/hostreporter/versionreportertest.cpp index 43c6e64b0de..617807699fa 100644 --- a/storage/src/tests/common/hostreporter/versionreportertest.cpp +++ b/storage/src/tests/common/hostreporter/versionreportertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/common/hostreporter/versionreporter.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/common/storagelinktest.cpp b/storage/src/tests/common/storagelinktest.cpp index 34b774ac424..6935882ce0d 100644 --- a/storage/src/tests/common/storagelinktest.cpp +++ b/storage/src/tests/common/storagelinktest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <tests/common/storagelinktest.h> #include <iostream> #include <string> diff --git a/storage/src/tests/common/storagelinktest.h b/storage/src/tests/common/storagelinktest.h index 089ba885ef7..ad7d11fd13d 100644 --- a/storage/src/tests/common/storagelinktest.h +++ b/storage/src/tests/common/storagelinktest.h @@ -4,6 +4,7 @@ #include <cppunit/extensions/HelperMacros.h> #include <tests/common/dummystoragelink.h> +#include <vespa/fastos/thread.h> namespace storage { diff --git a/storage/src/tests/common/testhelper.cpp b/storage/src/tests/common/testhelper.cpp index 3ec189c573b..d7d0f5eab24 100644 --- a/storage/src/tests/common/testhelper.cpp +++ b/storage/src/tests/common/testhelper.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <tests/common/testhelper.h> #include <vespa/log/log.h> diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp index 135d5c65110..7a390788bc1 100644 --- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <string> #include <sstream> diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index e35f147d0c8..fba4ad255e0 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -10,9 +10,7 @@ #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/simpleclusterinformation.h> #include <vespa/storage/distributor/distributor.h> - -#include <iostream> -#include <fstream> +#include <vespa/vespalib/text/stringtokenizer.h> using namespace storage::api; using namespace storage::lib; diff --git a/storage/src/tests/distributor/bucketgctimecalculatortest.cpp b/storage/src/tests/distributor/bucketgctimecalculatortest.cpp index 39bef3ec395..af61a9e434c 100644 --- a/storage/src/tests/distributor/bucketgctimecalculatortest.cpp +++ b/storage/src/tests/distributor/bucketgctimecalculatortest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <chrono> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/bucketgctimecalculator.h> diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 298a4283bfa..ea2a23743fe 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -12,6 +12,7 @@ #include <vespa/storage/config/config-stor-distributormanager.h> #include <tests/common/dummystoragelink.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/vespalib/text/stringtokenizer.h> namespace storage { diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index 7c709eed097..182bdfd2196 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -2,6 +2,7 @@ #include "distributortestutil.h" #include <vespa/storage/distributor/distributor.h> #include <vespa/config-stor-distribution.h> +#include <vespa/vespalib/text/stringtokenizer.h> namespace storage::distributor { diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp index 52ae9f6c8a6..4de158bc747 100644 --- a/storage/src/tests/distributor/maintenanceschedulertest.cpp +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <string> #include <sstream> diff --git a/storage/src/tests/distributor/mergelimitertest.cpp b/storage/src/tests/distributor/mergelimitertest.cpp index 57b3690dc24..04dfa1b4e7b 100644 --- a/storage/src/tests/distributor/mergelimitertest.cpp +++ b/storage/src/tests/distributor/mergelimitertest.cpp @@ -3,8 +3,7 @@ #include <vespa/storage/distributor/operations/idealstate/mergelimiter.h> #include <vespa/vdstestlib/cppunit/macros.h> -namespace storage { -namespace distributor { +namespace storage::distributor { struct MergeLimiterTest : public CppUnit::TestFixture { @@ -14,6 +13,10 @@ struct MergeLimiterTest : public CppUnit::TestFixture void testAllUntrustedLessThanMaxVariants(); void testAllUntrustedMoreThanMaxVariants(); void testSourceOnlyLast(); + void limited_set_cannot_be_just_source_only(); + void non_source_only_replica_chosen_from_in_sync_group(); + void non_source_only_replicas_preferred_when_replicas_not_in_sync(); + void at_least_one_non_source_only_replica_chosen_when_all_trusted(); CPPUNIT_TEST_SUITE(MergeLimiterTest); CPPUNIT_TEST(testKeepsAllBelowLimit); @@ -22,6 +25,10 @@ struct MergeLimiterTest : public CppUnit::TestFixture CPPUNIT_TEST(testAllUntrustedLessThanMaxVariants); CPPUNIT_TEST(testAllUntrustedMoreThanMaxVariants); CPPUNIT_TEST(testSourceOnlyLast); + CPPUNIT_TEST(limited_set_cannot_be_just_source_only); + CPPUNIT_TEST(non_source_only_replica_chosen_from_in_sync_group); + CPPUNIT_TEST(non_source_only_replicas_preferred_when_replicas_not_in_sync); + CPPUNIT_TEST(at_least_one_non_source_only_replica_chosen_when_all_trusted); CPPUNIT_TEST_SUITE_END(); }; @@ -56,12 +63,13 @@ namespace { #define ASSERT_LIMIT(maxNodes, nodes, result) \ { \ MergeLimiter limiter(maxNodes); \ - limiter.limitMergeToMaxNodes(nodes); \ + auto nodesCopy = nodes; \ + limiter.limitMergeToMaxNodes(nodesCopy); \ std::ostringstream actual; \ - for (uint32_t i=0; i<nodes.size(); ++i) { \ + for (uint32_t i = 0; i < nodesCopy.size(); ++i) { \ if (i != 0) actual << ","; \ - actual << nodes[i]._nodeIndex; \ - if (nodes[i]._sourceOnly) actual << 's'; \ + actual << nodesCopy[i]._nodeIndex; \ + if (nodesCopy[i]._sourceOnly) actual << 's'; \ } \ CPPUNIT_ASSERT_EQUAL(std::string(result), actual.str()); \ } @@ -153,8 +161,51 @@ MergeLimiterTest::testSourceOnlyLast() .add(13, 0x9) .add(1, 0x7) .add(4, 0x5)); - ASSERT_LIMIT(4, nodes, "13,1,2s,5s"); + ASSERT_LIMIT(4, nodes, "9,3,5s,2s"); } -} // distributor -} // storage +void MergeLimiterTest::limited_set_cannot_be_just_source_only() { + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(9, 0x6) + .addTrusted(2, 0x6) + .addTrusted(13, 0x6).setSourceOnly() + .add(1, 0x7).setSourceOnly()); + ASSERT_LIMIT(2, nodes, "2,13s"); + ASSERT_LIMIT(3, nodes, "2,13s,1s"); +} + +void MergeLimiterTest::non_source_only_replica_chosen_from_in_sync_group() { + // nodes 9, 2, 13 are all in sync. Merge limiter will currently by default + // pop the _last_ node of an in-sync replica "group" when outputting a limited + // set. Unless we special-case source-only replicas here, we'd end up with an + // output set of "13s,1s", i.e. all source-only. + MergeLimiter::NodeArray nodes(NodeFactory() + .add(9, 0x6) + .add(2, 0x6) + .add(13, 0x6).setSourceOnly() + .add(1, 0x7).setSourceOnly()); + ASSERT_LIMIT(2, nodes, "2,13s"); + ASSERT_LIMIT(3, nodes, "2,13s,1s"); +} + +void MergeLimiterTest::non_source_only_replicas_preferred_when_replicas_not_in_sync() { + MergeLimiter::NodeArray nodes(NodeFactory() + .add(9, 0x4) + .add(2, 0x5) + .add(13, 0x6).setSourceOnly() + .add(1, 0x7).setSourceOnly()); + ASSERT_LIMIT(2, nodes, "9,2"); + ASSERT_LIMIT(3, nodes, "9,2,13s"); +} + +void MergeLimiterTest::at_least_one_non_source_only_replica_chosen_when_all_trusted() { + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(9, 0x6) + .addTrusted(2, 0x6) + .addTrusted(13, 0x6).setSourceOnly() + .addTrusted(1, 0x6).setSourceOnly()); + ASSERT_LIMIT(2, nodes, "2,13s"); + ASSERT_LIMIT(3, nodes, "2,13s,1s"); +} + +} // storage::distributor diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 1e317e20b74..329776b2e09 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -9,6 +9,7 @@ #include <vespa/storage/distributor/bucketdbupdater.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/vespalib/text/stringtokenizer.h> using std::shared_ptr; @@ -26,6 +27,7 @@ class MergeOperationTest : public CppUnit::TestFixture, CPPUNIT_TEST(testDoNotRemoveActiveSourceOnlyCopies); CPPUNIT_TEST(testMarkRedundantTrustedCopiesAsSourceOnly); CPPUNIT_TEST(onlyMarkRedundantRetiredReplicasAsSourceOnly); + CPPUNIT_TEST(mark_post_merge_redundant_replicas_source_only); CPPUNIT_TEST_SUITE_END(); std::unique_ptr<PendingMessageTracker> _pendingTracker; @@ -38,6 +40,7 @@ protected: void testDoNotRemoveActiveSourceOnlyCopies(); void testMarkRedundantTrustedCopiesAsSourceOnly(); void onlyMarkRedundantRetiredReplicasAsSourceOnly(); + void mark_post_merge_redundant_replicas_source_only(); public: void setUp() override { @@ -269,13 +272,13 @@ MergeOperationTest::testGenerateNodeList() std::string("3,5,7,6,8,0,9,2,1,4"), getNodeList("storage:10", 10, "9,8,7,6,5,4,3,2,1,0")); - // Trusted copies should not be source only. + // Trusted copies can be source-only if they are in the non-ideal node set. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8,0,9,1,2,4s"), + std::string("3,5,7,6,8,0,9,1s,2s,4s"), getNodeList("storage:10", 7, "0,1t,2t,3,4,5,6,7,8,9")); CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8,0,9,2,1s,4s"), + std::string("3,5,7,6,8,0,9,1s,2s,4s"), getNodeList("storage:10", 7, "0,1,2t,3,4,5,6,7,8,9")); // Retired nodes are not in ideal state @@ -376,35 +379,31 @@ MergeOperationTest::testMarkRedundantTrustedCopiesAsSourceOnly() std::string("3,5,7,6s,8s"), getNodeList("storage:10", 3, "3t,5t,7t,6,8")); - // 3 redundancy, 4 trusted -> 1 source only trusted. - // We allow marking a trusted, non-ideal copy as source even when we don't - // have #redundancy trusted _ideal_ copies, as long as we're left with >= - // #redundancy trusted copies in total. + // Trusted-ness should not be taken into account when marking nodes as source-only. + // 2 out of 3 ideal replicas trusted. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8s"), + std::string("3,5,7,6s,8s"), getNodeList("storage:10", 3, "3t,5t,7,6t,8t")); - // Not sufficient number of trusted copies to mark any as source only. + // 1 out of 3 ideal replicas trusted. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8"), + std::string("3,5,7,6s,8s"), getNodeList("storage:10", 3, "3t,5,7,6t,8t")); - // Same as above, with all trusted copies being non-ideal. + // 0 out of 3 ideal replicas trusted. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8"), + std::string("3,5,7,6s,8s"), getNodeList("storage:10", 3, "3,5,7,6t,8t")); - // #redundancy of trusted, but none are ideal. Non-ideal trusted should - // not be marked as source only (though we can mark non-trusted non-ideal - // node as source only). - // Note the node reordering since trusted are added before the rest. + // #redundancy of trusted, but none are ideal. Non-ideal trusted may be + // marked as source only. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,8,0,9,6s"), + std::string("3,5,7,6s,8s,0s,9s"), getNodeList("storage:10", 3, "3,5,7,6,8t,0t,9t")); - // But allow for removing excess trusted, non-ideal copies. + // Allow for removing excess trusted, non-ideal copies. CPPUNIT_ASSERT_EQUAL( - std::string("3,5,7,6,8,0,9s"), + std::string("3,5,7,6s,8s,0s,9s"), getNodeList("storage:10", 3, "3,5,7,6t,8t,0t,9t")); } @@ -417,8 +416,41 @@ MergeOperationTest::onlyMarkRedundantRetiredReplicasAsSourceOnly() // source-only nodes will have their replica removed after a successful // merge, which we cannot allow to happen here. CPPUNIT_ASSERT_EQUAL( - std::string("0,1,2s"), - getNodeList("storage:3 .0.s.:r .1.s:r .2.s:r", 2, "1,0,2")); + std::string("1,0,2s"), + getNodeList("storage:3 .0.s:r .1.s:r .2.s:r", 2, "1,0,2")); +} + +void MergeOperationTest::mark_post_merge_redundant_replicas_source_only() { + // Ideal state sequence is [3, 5, 7, 6, 8, 0, 9, 2, 1, 4] + + // Retired node 7 is not part of the #redundancy ideal state and should be moved + // to node 6. Once the merge is done we'll end up with too many replicas unless + // we allow marking the to-be-moved replica as source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,6,7s"), + getNodeList("storage:10 .7.s:r", 3, "3t,5t,7t,6")); + + // Should be allowed to mark as source only even if retired replica is the + // only trusted replica at the time the merge starts. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,6,7s"), + getNodeList("storage:10 .7.s:r", 3, "3,5,7t,6")); + + // This extends to multiple retired nodes. + CPPUNIT_ASSERT_EQUAL( + std::string("3,6,8,5s,7s"), + getNodeList("storage:10 .5.s:r .7.s:r", 3, "3t,5t,7t,6,8")); + + // If number of post-merge ideal nodes is lower than desired redundancy, don't + // mark any as source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6"), + getNodeList("storage:10", 5, "3,5,7,6")); + + // Same applies to when post-merge ideal nodes is _equal_ to desired redundancy. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6"), + getNodeList("storage:10", 4, "3,5,7,6")); } } // distributor diff --git a/storage/src/tests/distributor/nodeinfotest.cpp b/storage/src/tests/distributor/nodeinfotest.cpp index 883e6ba7668..b702ed65f38 100644 --- a/storage/src/tests/distributor/nodeinfotest.cpp +++ b/storage/src/tests/distributor/nodeinfotest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <iomanip> #include <iostream> diff --git a/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp b/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp index f1c177e7939..ece6ce830a5 100644 --- a/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp +++ b/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> diff --git a/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp index a066649477c..c02b0d6df65 100644 --- a/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp +++ b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <string> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 342d346b398..9da9f7efc8b 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -5,6 +5,7 @@ #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> +#include <vespa/vespalib/text/stringtokenizer.h> namespace storage { diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 43d13acb28b..d5ab61087fb 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -3,10 +3,11 @@ #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <vespa/storageframework/generic/status/xmlstatusreporter.h> #include <tests/common/teststorageapp.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/document/util/stringutil.h> -#include <sstream> namespace storage { diff --git a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp index f47cc334e30..021188ac8d1 100644 --- a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp +++ b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/persistence/diskmoveoperationhandler.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/persistence/messages.h> diff --git a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp index c4e3ef97eb2..5e65f38e5b5 100644 --- a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> diff --git a/storage/src/tests/persistence/filestorage/deletebuckettest.cpp b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp index d415db1b45e..4c81d96603f 100644 --- a/storage/src/tests/persistence/filestorage/deletebuckettest.cpp +++ b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index e2f41aa5b3e..52b976586e8 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -11,6 +11,7 @@ #include <vespa/vdslib/container/mutabledocumentlist.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/multioperation.h> diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp index fd106fb3117..189ffffa52c 100644 --- a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp +++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vector> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/persistence/messages.h> diff --git a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp index 363c89165b3..aa6aefccfbf 100644 --- a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp +++ b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <tests/persistence/common/persistenceproviderwrapper.h> diff --git a/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp index c754ca353af..28e82149b37 100644 --- a/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp +++ b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index 3a262c01d54..bc39f1aa799 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index f50e62e0aeb..e28dd3380dd 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/persistence/persistencethread.h> #include <vespa/storageapi/message/bucketsplitting.h> diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index db75725db6f..42d13bb6240 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/storage/persistence/processallhandler.h> #include <vespa/vdstestlib/cppunit/macros.h> diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp index 65ea629ad81..47efaaa371e 100644 --- a/storage/src/tests/persistence/splitbitdetectortest.cpp +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <fstream> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/storage/persistence/splitbitdetector.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index fe062a9ee30..c01b24aae8d 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/storageserver/communicationmanager.h> #include <vespa/messagebus/testlib/slobrok.h> diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 1f1198efc3c..26317465b5a 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -2,14 +2,15 @@ #include <vespa/document/base/testdocrepo.h> #include <cppunit/extensions/HelperMacros.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/messagebus/emptyreply.h> #include <vespa/storage/storageserver/documentapiconverter.h> #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/persistence.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/testkit/test_kit.h> using document::DataType; diff --git a/storage/src/tests/storageserver/dummystoragelink.cpp b/storage/src/tests/storageserver/dummystoragelink.cpp index 7194f1fba3d..4ccaedbc60f 100644 --- a/storage/src/tests/storageserver/dummystoragelink.cpp +++ b/storage/src/tests/storageserver/dummystoragelink.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <tests/common/dummystoragelink.h> #include <sys/time.h> diff --git a/storage/src/tests/storageutil/functortest.cpp b/storage/src/tests/storageutil/functortest.cpp index 00b9f5450cb..2e0d6c397fc 100644 --- a/storage/src/tests/storageutil/functortest.cpp +++ b/storage/src/tests/storageutil/functortest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <cppunit/extensions/HelperMacros.h> #include <list> #include <string> diff --git a/storage/src/tests/visiting/memory_bounded_trace_test.cpp b/storage/src/tests/visiting/memory_bounded_trace_test.cpp index 85eae12fc34..e2c44cf0b74 100644 --- a/storage/src/tests/visiting/memory_bounded_trace_test.cpp +++ b/storage/src/tests/visiting/memory_bounded_trace_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/visiting/memory_bounded_trace.h> diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index db4cfcc7ecb..79150c160d5 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -7,16 +7,20 @@ #include <iomanip> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/storageutil/distributorstatecache.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <vespa/storageframework/generic/status/xmlstatusreporter.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/stat.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/config/config.h> #include <unordered_map> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.manager"); namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index e0f3ed5a556..09cbc60b58e 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -18,11 +18,14 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/generic/status/statusreporter.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> + #include <vespa/storageapi/message/bucket.h> #include <vespa/config/subscription/configuri.h> #include <unordered_set> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp index 80f82e65a9a..39e45e1b541 100644 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp +++ b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "distribution_hash_normalizer.h" -#include <vespa/log/log.h> #include <vespa/vespalib/stllike/asciistream.h> #include <boost/spirit/include/qi.hpp> #include <boost/spirit/include/phoenix_core.hpp> @@ -13,6 +12,7 @@ #include <iterator> #include <functional> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.distribution_hash_normalizer"); // TODO diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp index 5fc4ece8191..9cee5b6acee 100644 --- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp @@ -2,6 +2,7 @@ #include "mapbucketdatabase.h" #include <vespa/storage/common/bucketoperationlogger.h> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 178731469ba..49022db523b 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -13,7 +13,7 @@ #include <vespa/config/helper/configgetter.hpp> #include <iomanip> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.initializer"); namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index 4c87c39fdd2..92c00c4420a 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -44,11 +44,13 @@ #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelink.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vdslib/state/nodestate.h> #include <vespa/config/subscription/configuri.h> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp index e4ddc3cb5a7..b38c061de44 100644 --- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp +++ b/storage/src/vespa/storage/bucketmover/bucketmover.cpp @@ -6,11 +6,12 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/storageutil/log.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketmover"); -namespace storage { -namespace bucketmover { +namespace storage::bucketmover { BucketMover::BucketMover(const config::ConfigUri & configUri, ServiceLayerComponentRegister& reg) @@ -525,5 +526,4 @@ BucketMover::printRunStatisticsHtml(std::ostream& out, rs.print(out, true, ""); } -} // bucketmover -} // storage +} diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.h b/storage/src/vespa/storage/bucketmover/bucketmover.h index a3473f778c4..4219bc0dac6 100644 --- a/storage/src/vespa/storage/bucketmover/bucketmover.h +++ b/storage/src/vespa/storage/bucketmover/bucketmover.h @@ -20,9 +20,8 @@ #include <vespa/storage/common/storagelink.h> #include <vespa/storage/config/config-stor-bucketmover.h> #include <vespa/storage/common/servicelayercomponent.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/config/helper/ifetchercallback.h> #include <vespa/config/subscription/configuri.h> @@ -31,7 +30,6 @@ namespace storage { class BucketDiskMoveCommand; -class Clock; namespace bucketmover { diff --git a/storage/src/vespa/storage/bucketmover/run.cpp b/storage/src/vespa/storage/bucketmover/run.cpp index e202ddfba58..dbfa590bd43 100644 --- a/storage/src/vespa/storage/bucketmover/run.cpp +++ b/storage/src/vespa/storage/bucketmover/run.cpp @@ -3,6 +3,7 @@ #include "run.h" #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/bucketdb/lockablemap.hpp> +#include <vespa/storageframework/generic/clock/clock.h> #include <iomanip> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.cpp b/storage/src/vespa/storage/bucketmover/runstatistics.cpp index 093d5290203..8daa0ffaee8 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.cpp +++ b/storage/src/vespa/storage/bucketmover/runstatistics.cpp @@ -2,9 +2,10 @@ #include "runstatistics.h" #include "htmltable.h" +#include <vespa/storageframework/generic/clock/clock.h> -namespace storage { -namespace bucketmover { + +namespace storage::bucketmover { RunStatistics::DiskMatrix::DiskMatrix() : _bucketsMoved(0), @@ -185,5 +186,4 @@ RunStatistics::getBucketCount(uint16_t disk, bool includeWrongLocation) const return total; } -} // bucketmover -} // storage +} diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.h b/storage/src/vespa/storage/bucketmover/runstatistics.h index 2494c45241f..d14ae161b75 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.h +++ b/storage/src/vespa/storage/bucketmover/runstatistics.h @@ -36,18 +36,15 @@ } \ } -#include <vespa/storageframework/storageframework.h> #include <vespa/vdslib/state/nodestate.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/printable.h> -#include <vector> - -namespace storage { +#include <vespa/storageframework/generic/clock/time.h> -class Clock; +#include <vector> -namespace bucketmover { +namespace storage::bucketmover { struct RunStatistics : public document::Printable { using DiskDistribution = lib::Distribution::DiskDistribution; @@ -95,5 +92,4 @@ struct RunStatistics : public document::Printable { void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; -} // bucketmover -} // storage +} diff --git a/storage/src/vespa/storage/common/bucketoperationlogger.h b/storage/src/vespa/storage/common/bucketoperationlogger.h index 12a07ccf539..d4e13cf4099 100644 --- a/storage/src/vespa/storage/common/bucketoperationlogger.h +++ b/storage/src/vespa/storage/common/bucketoperationlogger.h @@ -8,7 +8,6 @@ #include <vespa/vespalib/util/sync.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/vstringfmt.h> -#include <vespa/storageframework/storageframework.h> /** * Enable this to log most slotfile operations (such as all mutations) as diff --git a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp index 59c0f1c36f3..76a38e3e730 100644 --- a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp @@ -1,14 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "cpureporter.h" - -#include <vespa/log/log.h> +#include "kernelmetrictool.h" #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/text/stringtokenizer.h> -#include "kernelmetrictool.h" -#include <array> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".cpureporter"); namespace storage { diff --git a/storage/src/vespa/storage/common/hostreporter/diskreporter.cpp b/storage/src/vespa/storage/common/hostreporter/diskreporter.cpp index 23db7137033..0faf400e6d7 100644 --- a/storage/src/vespa/storage/common/hostreporter/diskreporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/diskreporter.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "diskreporter.h" #include "kernelmetrictool.h" diff --git a/storage/src/vespa/storage/common/hostreporter/hostinfo.cpp b/storage/src/vespa/storage/common/hostreporter/hostinfo.cpp index f34b459fc51..4571c6883b3 100644 --- a/storage/src/vespa/storage/common/hostreporter/hostinfo.cpp +++ b/storage/src/vespa/storage/common/hostreporter/hostinfo.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "hostinfo.h" #include "hostreporter.h" diff --git a/storage/src/vespa/storage/common/hostreporter/memreporter.cpp b/storage/src/vespa/storage/common/hostreporter/memreporter.cpp index 862e1c6a8ed..cb66337ec4c 100644 --- a/storage/src/vespa/storage/common/hostreporter/memreporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/memreporter.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "memreporter.h" #include "kernelmetrictool.h" diff --git a/storage/src/vespa/storage/common/hostreporter/networkreporter.cpp b/storage/src/vespa/storage/common/hostreporter/networkreporter.cpp index 04a10699d0f..744cd583a8e 100644 --- a/storage/src/vespa/storage/common/hostreporter/networkreporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/networkreporter.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "networkreporter.h" #include "kernelmetrictool.h" #include <vespa/vespalib/text/stringtokenizer.h> diff --git a/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp b/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp index 670f4ef6369..5df7122ddec 100644 --- a/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp +++ b/storage/src/vespa/storage/common/hostreporter/versionreporter.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "versionreporter.h" #include <vespa/vespalib/component/vtag.h> diff --git a/storage/src/vespa/storage/common/statusmessages.cpp b/storage/src/vespa/storage/common/statusmessages.cpp index c42212f9a63..85a6690eb85 100644 --- a/storage/src/vespa/storage/common/statusmessages.cpp +++ b/storage/src/vespa/storage/common/statusmessages.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statusmessages.h" +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/common/statusmessages.h b/storage/src/vespa/storage/common/statusmessages.h index 4299fefe773..9c432c62790 100644 --- a/storage/src/vespa/storage/common/statusmessages.h +++ b/storage/src/vespa/storage/common/statusmessages.h @@ -7,7 +7,7 @@ #pragma once #include <vespa/storageapi/message/internal.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/httpurlpath.h> namespace storage { diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 638ff54cbc7..8cb5c9018f4 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -1,22 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statusmetricconsumer.h" - +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <boost/assign.hpp> #include <boost/lexical_cast.hpp> -#include <vespa/log/log.h> #include <vespa/metrics/printutils.h> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> #include <vespa/metrics/xmlwriter.h> -#include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/asciistream.h> - // For setlocale to print . separated numbers -#include <locale.h> -#include <stdio.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <vespa/log/log.h> LOG_SETUP(".status.metricreporter"); namespace storage { diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index 6a2c18f78bd..1812bed281c 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -10,11 +10,12 @@ #pragma once #include "storagecomponent.h" +#include <vespa/storageframework/generic/status/statusreporter.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> -#include <map> #include <vespa/metrics/metrics.h> -#include <vespa/storageframework/storageframework.h> +#include <map> namespace vespalib { class StringTokenizer; @@ -22,6 +23,8 @@ namespace vespalib { namespace storage { +namespace framework { class MemoryToken; } + class StatusMetricConsumer : public framework::StatusReporter, private framework::MetricUpdateHook, private vespalib::JsonStreamTypes @@ -50,7 +53,7 @@ private: vespalib::Monitor _waiter; framework::SecondTime _startTime; framework::SecondTime _processedTime; - framework::MemoryToken::UP _metricMemoryToken; + std::unique_ptr<framework::MemoryToken> _metricMemoryToken; void writeXmlTags(std::ostream& out, const vespalib::StringTokenizer& name, diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index d3b6370347b..50097b132f6 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/util/backtrace.h> #include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".application.link"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index c894b6196e8..b6935719cc5 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -15,13 +15,19 @@ #pragma once #include "storagelink.h" -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/vespalib/util/document_runnable.h> #include <deque> #include <limits> namespace storage { +namespace framework { + class ComponentRegister; + class Component; + class Thread; +} + class StorageLinkQueued : public StorageLink { public: StorageLinkQueued(const std::string& name, framework::ComponentRegister& cr); @@ -72,8 +78,8 @@ private: vespalib::Monitor _sync; std::deque< std::shared_ptr<Message> > _messages; bool _replyDispatcher; - framework::Component::UP _component; - framework::Thread::UP _thread; + std::unique_ptr<framework::Component> _component; + std::unique_ptr<framework::Thread> _thread; void terminate(); public: @@ -124,8 +130,8 @@ private: }; framework::ComponentRegister& _compReg; - framework::Thread::UP _replyThread; - framework::Thread::UP _commandThread; + std::unique_ptr<framework::Thread> _replyThread; + std::unique_ptr<framework::Thread> _commandThread; ReplyDispatcher _replyDispatcher; CommandDispatcher _commandDispatcher; uint16_t _closeState; diff --git a/storage/src/vespa/storage/common/storagelinkqueued.hpp b/storage/src/vespa/storage/common/storagelinkqueued.hpp index daca4bac5c5..d5a4c8fbdef 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.hpp +++ b/storage/src/vespa/storage/common/storagelinkqueued.hpp @@ -3,6 +3,8 @@ #pragma once #include "storagelinkqueued.h" +#include <vespa/storageframework/generic/thread/thread.h> +#include <vespa/storageframework/generic/component/component.h> #include <vespa/vespalib/util/stringfmt.h> #include <sstream> diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 10d135ccf2d..57e8c55c23b 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -3,6 +3,7 @@ #include <vespa/document/select/parser.h> #include <vespa/document/select/traversingvisitor.h> #include <vespa/vespalib/util/exceptions.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".distributorconfiguration"); diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 068f42f8b31..9051bbac0f0 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -7,8 +7,9 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/multioperation.h> +#include <vespa/vespalib/util/xmlstream.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.bucketdb.updater"); using storage::lib::Node; diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index a8f76c2c7a0..3bd86fc3db6 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -12,16 +12,14 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> -#include <vespa/storageframework/storageframework.h> - +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <set> #include <deque> +#include <list> -namespace storage { - -namespace distributor { +namespace storage::distributor { class Distributor; @@ -264,5 +262,3 @@ private: }; } - -} diff --git a/storage/src/vespa/storage/distributor/bucketgctimecalculator.cpp b/storage/src/vespa/storage/distributor/bucketgctimecalculator.cpp index 85627ddf38f..62a6ec4f3c8 100644 --- a/storage/src/vespa/storage/distributor/bucketgctimecalculator.cpp +++ b/storage/src/vespa/storage/distributor/bucketgctimecalculator.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketgctimecalculator.h> +#include "bucketgctimecalculator.h" -namespace storage { -namespace distributor { +namespace storage::distributor { bool BucketGcTimeCalculator::shouldGc(const document::BucketId& b, @@ -27,7 +25,4 @@ BucketGcTimeCalculator::shouldGc(const document::BucketId& b, return lastRunAt < (newestValid - _checkInterval); } - -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/bucketlistmerger.cpp b/storage/src/vespa/storage/distributor/bucketlistmerger.cpp index 894a8717c8a..02bcf47e56e 100644 --- a/storage/src/vespa/storage/distributor/bucketlistmerger.cpp +++ b/storage/src/vespa/storage/distributor/bucketlistmerger.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketlistmerger.h> + +#include "bucketlistmerger.h" using namespace storage::distributor; diff --git a/storage/src/vespa/storage/distributor/clusterinformation.cpp b/storage/src/vespa/storage/distributor/clusterinformation.cpp index b6d0b19cb2b..bf37b0051e1 100644 --- a/storage/src/vespa/storage/distributor/clusterinformation.cpp +++ b/storage/src/vespa/storage/distributor/clusterinformation.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/clusterinformation.h> + +#include "clusterinformation.h" #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -namespace storage { -namespace distributor { +namespace storage::distributor { bool ClusterInformation::ownsBucket(const document::BucketId& bucketId) const @@ -53,4 +52,3 @@ ClusterInformation::getStorageNodeCount() const } } -} diff --git a/storage/src/vespa/storage/distributor/delegatedstatusrequest.h b/storage/src/vespa/storage/distributor/delegatedstatusrequest.h index ca0fb239f76..d0c587f6691 100644 --- a/storage/src/vespa/storage/distributor/delegatedstatusrequest.h +++ b/storage/src/vespa/storage/distributor/delegatedstatusrequest.h @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/statusreporter.h> -namespace storage { -namespace distributor { +namespace storage::distributor { struct DelegatedStatusRequest { @@ -25,5 +24,4 @@ private: DelegatedStatusRequest& operator=(const DelegatedStatusRequest&); }; -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index fdda56cea55..944a93d5f04 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -1,18 +1,20 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/storage/distributor/distributor.h> +// +#include "distributor.h" +#include "blockingoperationstarter.h" +#include "throttlingoperationstarter.h" +#include "idealstatemetricsset.h" +#include "ownership_transfer_safe_time_point_calculator.h" +#include "managed_bucket_space_repo.h" #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> -#include <vespa/storage/distributor/blockingoperationstarter.h> -#include <vespa/storage/distributor/throttlingoperationstarter.h> -#include <vespa/storage/distributor/idealstatemetricsset.h> -#include <vespa/storage/distributor/ownership_transfer_safe_time_point_calculator.h> -#include <vespa/storage/distributor/managed_bucket_space_repo.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/hostreporter/hostinfo.h> +#include <vespa/storageframework/generic/status/xmlstatusreporter.h> + +#include <vespa/log/log.h> LOG_SETUP(".distributor-main"); namespace storage { diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 182988104a9..f47f4e4702c 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -19,6 +19,7 @@ #include <vespa/storage/common/messagesender.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/thread/tickingthread.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/config/config.h> #include <vespa/vespalib/util/sync.h> #include <unordered_map> diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp index 0c739c65425..915d90b5676 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/distributor_host_info_reporter.h> -#include <vespa/storage/distributor/min_replica_provider.h> -#include <vespa/storage/distributor/pendingmessagetracker.h> + +#include "distributor_host_info_reporter.h" +#include "min_replica_provider.h" +#include "pendingmessagetracker.h" #include <set> diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributorcomponent.h index 6f94d1bdf69..c91a7724eaa 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.h +++ b/storage/src/vespa/storage/distributor/distributorcomponent.h @@ -6,7 +6,6 @@ #include "statechecker.h" #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/storageutil/utils.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/buckets/bucketinfo.h> #include <vespa/vdslib/state/clusterstate.h> diff --git a/storage/src/vespa/storage/distributor/distributormessagesender.cpp b/storage/src/vespa/storage/distributor/distributormessagesender.cpp index d7b970baadd..efb18662ba2 100644 --- a/storage/src/vespa/storage/distributor/distributormessagesender.cpp +++ b/storage/src/vespa/storage/distributor/distributormessagesender.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/distributormessagesender.h> -#include <vespa/storageapi/messageapi/storagecommand.h> -namespace storage { +#include "distributormessagesender.h" +#include <vespa/storageapi/messageapi/storagecommand.h> -namespace distributor { +namespace storage::distributor { uint64_t DistributorMessageSender::sendToNode( @@ -27,7 +25,5 @@ DistributorMessageSender::sendToNode( return msgId; } - } -} diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 75ade3e2128..d5d7077c4c9 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -8,7 +8,6 @@ #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/managed_bucket_space_component.h> #include <vespa/storageapi/messageapi/messagehandler.h> -#include <vespa/storageframework/storageframework.h> #include <chrono> namespace storage { diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 3b468a64024..1245781a2c1 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -8,8 +8,6 @@ #include <vespa/storage/distributor/statechecker.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vdslib/state/clusterstate.h> #include <vector> diff --git a/storage/src/vespa/storage/distributor/latency_statistics_provider.cpp b/storage/src/vespa/storage/distributor/latency_statistics_provider.cpp index 7207c4745fc..adca5773f31 100644 --- a/storage/src/vespa/storage/distributor/latency_statistics_provider.cpp +++ b/storage/src/vespa/storage/distributor/latency_statistics_provider.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/latency_statistics_provider.h> + +#include "latency_statistics_provider.h" #include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp b/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp index 5dd9567044d..8865d0443bd 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> -#include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> + +#include "maintenancescheduler.h" +#include "maintenanceoperationgenerator.h" #include <vespa/storage/distributor/operationstarter.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> -namespace storage { -namespace distributor { +namespace storage::distributor { MaintenanceScheduler::MaintenanceScheduler( MaintenanceOperationGenerator& operationGenerator, @@ -102,4 +101,3 @@ MaintenanceScheduler::startOperation(const PrioritizedBucket& bucket) } } -} diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp index f107efc8d93..7a873a73bb2 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> + +#include "node_maintenance_stats_tracker.h" #include <ostream> -namespace storage { -namespace distributor { +namespace storage::distributor { const NodeMaintenanceStats NodeMaintenanceStatsTracker::_emptyStats; @@ -23,6 +22,5 @@ operator<<(std::ostream& os, const NodeMaintenanceStats& stats) NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker() {} NodeMaintenanceStatsTracker::~NodeMaintenanceStatsTracker() {} -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.cpp b/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.cpp index 9f6c14a171a..72cac9b882e 100644 --- a/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + +#include "prioritizedbucket.h" #include <iostream> -#include <vespa/storage/distributor/maintenance/prioritizedbucket.h> -namespace storage { -namespace distributor { +namespace storage::distributor { const PrioritizedBucket PrioritizedBucket::INVALID = PrioritizedBucket(); @@ -16,4 +15,3 @@ operator<<(std::ostream& os, const PrioritizedBucket& bucket) } } -} diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp index 93369faa32e..23774cbc3c1 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> + +#include "simplebucketprioritydatabase.h" #include <iostream> #include <sstream> -namespace storage { -namespace distributor { +namespace storage::distributor { SimpleBucketPriorityDatabase::~SimpleBucketPriorityDatabase() { @@ -140,4 +139,3 @@ SimpleBucketPriorityDatabase::toString() const } } -} diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp b/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp index f847390a0ce..8f5055da557 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp +++ b/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/managed_bucket_space_component.h> +#include "managed_bucket_space_component.h" -namespace storage { -namespace distributor { +namespace storage::distributor { ManagedBucketSpaceComponent::ManagedBucketSpaceComponent( DistributorInterface& distributor, @@ -16,5 +14,4 @@ ManagedBucketSpaceComponent::ManagedBucketSpaceComponent( { } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/messageguard.h b/storage/src/vespa/storage/distributor/messageguard.h index 8d549965136..2944cc20c70 100644 --- a/storage/src/vespa/storage/distributor/messageguard.h +++ b/storage/src/vespa/storage/distributor/messageguard.h @@ -1,8 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "pendingclusterstate.h" #include <vespa/storage/common/messagesender.h> -#include <vespa/storage/distributor/pendingclusterstate.h> +#include <vespa/vespalib/util/sync.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index aeca85e821d..0b213a85dd3 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -33,7 +33,7 @@ MessageTracker::handleReply(api::BucketReply& reply) { std::map<uint64_t, uint16_t>::iterator found = _sentMessages.find(reply.getMsgId()); if (found == _sentMessages.end()) { - LOG(warning, "Received reply %" PRIu64 " for callback which we have no recollection of", reply.getMsgId()); + LOG(warning, "Received reply %zu for callback which we have no recollection of", reply.getMsgId()); return (uint16_t)-1; } else { uint16_t node = found->second; diff --git a/storage/src/vespa/storage/distributor/nodeinfo.cpp b/storage/src/vespa/storage/distributor/nodeinfo.cpp index 52c16dbb08d..1fcb804647f 100644 --- a/storage/src/vespa/storage/distributor/nodeinfo.cpp +++ b/storage/src/vespa/storage/distributor/nodeinfo.cpp @@ -1,10 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/nodeinfo.h> -namespace storage { +#include "nodeinfo.h" +#include <vespa/storageframework/generic/clock/clock.h> -namespace distributor { +namespace storage::distributor { NodeInfo::NodeInfo(const framework::Clock& clock) : _clock(clock) {} @@ -81,5 +80,3 @@ NodeInfo::getNode(uint16_t idx) const } } - -} diff --git a/storage/src/vespa/storage/distributor/nodeinfo.h b/storage/src/vespa/storage/distributor/nodeinfo.h index 59943ed494a..c7e4982c5b5 100644 --- a/storage/src/vespa/storage/distributor/nodeinfo.h +++ b/storage/src/vespa/storage/distributor/nodeinfo.h @@ -7,11 +7,10 @@ */ #pragma once -#include <vespa/storageframework/storageframework.h> #include <vector> +#include <vespa/storageframework/generic/clock/time.h> -namespace storage { -namespace distributor { +namespace storage::distributor { class NodeInfo { public: @@ -45,6 +44,4 @@ private: SingleNodeInfo& getNode(uint16_t idx); }; -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/operationowner.cpp b/storage/src/vespa/storage/distributor/operationowner.cpp index 4ec21742cbf..2157d3aee9c 100644 --- a/storage/src/vespa/storage/distributor/operationowner.cpp +++ b/storage/src/vespa/storage/distributor/operationowner.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/operations/operation.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/messageapi/storagereply.h> +#include <vespa/storageframework/generic/clock/clock.h> #include <vespa/log/log.h> LOG_SETUP(".operationowner"); diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index bbbcc4dd7e4..05c90ad9584 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -1,17 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/operations/external/getoperation.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/log/log.h> +#include "getoperation.h" #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/state/nodestate.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/log/log.h> LOG_SETUP(".distributor.callback.doc.get"); -namespace storage { -namespace distributor { +namespace storage::distributor { GetOperation::GroupId::GroupId(const document::BucketId& id, uint32_t checksum, @@ -283,6 +281,4 @@ GetOperation::hasConsistentCopies() const return _responses.size() == 1; } -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index 57a7c55fb81..c51291f1b7d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -5,6 +5,7 @@ #include <vespa/storage/distributor/operations/operation.h> #include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageframework/generic/clock/timer.h> namespace document { class Document; } diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 5b907d21fcd..54ef503a772 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -5,6 +5,7 @@ #include "putoperation.h" #include "updateoperation.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/select/parser.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/batch.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index d2b29a092bc..ec4dd75197b 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/operations/external/updateoperation.h> -#include <vespa/document/fieldvalue/document.h> + +#include "updateoperation.h" +#include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/log/log.h> -#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/log/log.h> LOG_SETUP(".distributor.callback.doc.update"); diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index 18ac302c019..feab573cb75 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -7,7 +7,7 @@ #include <vespa/storageapi/defs.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/message/visitor.h> - +#include <vespa/storageframework/generic/clock/timer.h> namespace document { class Document; } @@ -183,5 +183,3 @@ private: } } - - diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 257e9e53d4f..22c71d60f9f 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> +#include "idealstateoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/log/log.h> -#include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/distributor/pendingmessagetracker.h> +#include <vespa/storageapi/messageapi/maintenancecommand.h> +#include <vespa/log/log.h> LOG_SETUP(".distributor.operation"); using namespace storage; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index 5c1f906aae7..e2417ea1585 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -1,14 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/operations/idealstate/joinoperation.h> -#include <vespa/storageapi/messageapi/storagereply.h> +#include "joinoperation.h" #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/vdslib/state/clusterstate.h> -#include <vespa/storage/distributor/pendingmessagetracker.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.join"); using namespace storage::distributor; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.cpp index 618ba9ba884..d708b902454 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.cpp @@ -1,20 +1,32 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/storage/distributor/operations/idealstate/mergelimiter.h> +#include <cassert> #include <vespa/log/log.h> - LOG_SETUP(".distributor.operations.merge.limiter"); -namespace storage { -namespace distributor { +namespace storage::distributor { MergeLimiter::MergeLimiter(uint16_t maxNodes) : _maxNodes(maxNodes) { + assert(maxNodes > 1); LOG(spam, "Limiter initialized with %u nodes.", uint32_t(maxNodes)); } +// TODO replace this overly complicated set of heuristics with something simpler. +// Suggestion: +// 1. Find non-source only replica with highest meta entry count. Emit it and remove from set. +// This tries to maintain a "seed" replica that can hopefully let the remaining replicas +// converge to the complete document entry set as quickly as possible. +// 2. Create mapping from checksum -> replica set. +// 3. Circularly loop through mapping and emit+remove the first replica in each mapping's set. +// Distributing the merge across replica checksum groups is a heuristic to fetch as many +// distinct document entries in one merge operation as possible, as these are all known to +// be pairwise divergent from each other. +// 3.1 Once merge limit is reached, break +// 4. Do a stable sort on the emitted list such that source only replicas are last in the sequence. namespace { class EqualCopies { uint32_t _checksum; @@ -28,15 +40,24 @@ namespace { { } - bool hasTrusted() const { return (_trustedCopies > 0); } - uint32_t trustedCount() const { return _trustedCopies; } - uint32_t size() const { return _copies.size(); } - bool operator==(const MergeMetaData& mmd) const { + bool hasTrusted() const noexcept { return (_trustedCopies > 0); } + uint32_t trustedCount() const noexcept { return _trustedCopies; } + uint32_t size() const noexcept { return static_cast<uint32_t>(_copies.size()); } + bool operator==(const MergeMetaData& mmd) const noexcept { return (_checksum == mmd.checksum()); } void add(const MergeMetaData& mmd) { - if (_copies.empty()) _checksum = mmd.checksum(); - if (mmd.trusted()) ++_trustedCopies; + if (_copies.empty()) { + _checksum = mmd.checksum(); + } + // Don't treat source only replicas as trusted from the perspective of + // picking replica groups. "Trusted" in the context of the merge limiter + // logic _in practice_ means "may be output as the sole non-source only node + // in the resulting node set", which obviously doesn't work if it is in fact + // source only to begin with... + if (mmd.trusted() && !mmd.source_only()) { + ++_trustedCopies; + } _copies.push_back(mmd); } MergeMetaData extractNext() { @@ -56,36 +77,22 @@ namespace { : _trustedCopies(0) { _groups.reserve(a.size()); - for (uint32_t i=0, n=a.size(); i<n; ++i) { + for (uint32_t i = 0, n = static_cast<uint32_t>(a.size()); i < n; ++i) { add(a[i]); - if (a[i].trusted()) { + if (a[i].trusted() && !a[i].source_only()) { ++_trustedCopies; } } } - EqualCopies& getMajority() { - EqualCopies* candidate = 0; - uint32_t size = 0; - for (uint32_t i=0, n=_groups.size(); i<n; ++i) { - if (_groups[i].size() > size) { - candidate = &_groups[i]; - size = candidate->size(); - } - } - assert(candidate != 0); - return *candidate; - } - - bool hasTrusted() const { return (_trustedCopies > 0); } - uint32_t trustedCount() const { return _trustedCopies; } + bool hasTrusted() const noexcept { return (_trustedCopies > 0); } Statistics extractGroupsWithTrustedCopies() { std::vector<EqualCopies> remaining; Statistics trusted; remaining.reserve(_groups.size()); trusted._groups.reserve(_groups.size()); - for (uint32_t i=0, n=_groups.size(); i<n; ++i) { + for (uint32_t i = 0, n = static_cast<uint32_t>(_groups.size()); i < n; ++i) { if (_groups[i].hasTrusted()) { trusted._groups.push_back(_groups[i]); trusted._trustedCopies += _groups[i].trustedCount(); @@ -110,7 +117,7 @@ namespace { void removeGroup(uint32_t groupIndex) { std::vector<EqualCopies> remaining; remaining.reserve(_groups.size()-1); - for (uint32_t i=0, n=_groups.size(); i<n; ++i) { + for (uint32_t i = 0, n = static_cast<uint32_t>(_groups.size()); i < n; ++i) { if (i != groupIndex) { remaining.push_back(_groups[i]); } @@ -120,10 +127,16 @@ namespace { private: void add(const MergeMetaData& mmd) { - for (uint32_t i=0; i<_groups.size(); ++i) { - if (_groups[i] == mmd) { - _groups[i].add(mmd); - return; + // Treat source only replicas as their own distinct "groups" with regards + // to picking replicas for being part of the merge. This way, we avoid + // accidentally picking a trusted source only replica as our one trusted + // replica that will be part of the merge. + if (!mmd.source_only()) { + for (uint32_t i = 0; i < _groups.size(); ++i) { + if (_groups[i] == mmd) { + _groups[i].add(mmd); + return; + } } } _groups.push_back(EqualCopies()); @@ -131,13 +144,15 @@ namespace { } }; - // Add up to max nodes, where different variants exist, prefer having - // some of each. + // Add up to max nodes, where different variants exist, prefer having + // some of each. void addNodes(uint32_t max, Statistics& stats, MergeLimiter::NodeArray& result) { + // FIXME redesign! `last` will unsigned over/underflow in extractNext, which + // is not a very pretty solution, to say the least. uint32_t last = -1; - for (uint32_t i=0; i<max; ++i) { + for (uint32_t i = 0; i < max; ++i) { MergeMetaData data; if (!stats.extractNext(data, last)) return; result.push_back(data); @@ -152,16 +167,22 @@ namespace { }; } +// FIXME the only reason why this code doesn't end up accidentally picking +// just source-only replicas as the output node set today is due to an implicit +// guarantee that the input to this function always has source-only replicas +// listed _last_ in the sequence. void MergeLimiter::limitMergeToMaxNodes(NodeArray& nodes) { - // If not above max anyhow, we need not do anything - if (nodes.size() <= _maxNodes) return; - // Gather some statistics to base decision on what we are going to do on + // If not above max anyhow, we need not do anything + if (nodes.size() <= _maxNodes) { + return; + } + // Gather some statistics to base decision on what we are going to do on Statistics stats(nodes); NodeArray result; - // If we have trusted copies, these should be complete. Pick one of them - // and merge with as many untrusted copies as possible + // If we have trusted copies, these are likely to be complete. Pick one of them + // and merge with as many untrusted copies as possible if (stats.hasTrusted()) { Statistics trusted(stats.extractGroupsWithTrustedCopies()); addNodes(_maxNodes - 1, stats, result); @@ -173,5 +194,4 @@ MergeLimiter::limitMergeToMaxNodes(NodeArray& nodes) result.swap(nodes); } -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.h index acde58e6061..ef86fc07810 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergelimiter.h @@ -5,8 +5,7 @@ #include <vespa/storage/distributor/operations/idealstate/mergemetadata.h> #include <vector> -namespace storage { -namespace distributor { +namespace storage::distributor { class MergeLimiter { uint16_t _maxNodes; @@ -19,5 +18,4 @@ public: void limitMergeToMaxNodes(NodeArray&); }; -} // distributor -} // storage +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h index 8c1cb02bcda..4c4225587bc 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h @@ -25,6 +25,7 @@ struct MergeMetaData { assert(_copy != 0); return _copy->getChecksum(); } + bool source_only() const noexcept { return _sourceOnly; } }; vespalib::asciistream& operator<<(vespalib::asciistream& out, const MergeMetaData& e); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 75c58ca4f87..80a79a8cde9 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -2,7 +2,7 @@ #include "mergeoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.merge"); namespace storage::distributor { @@ -20,18 +20,13 @@ MergeOperation::getStatus() const void MergeOperation::addIdealNodes( - const lib::Distribution& distribution, - const lib::ClusterState& state, - const document::BucketId& bucketId, + const std::vector<uint16_t>& idealNodes, const std::vector<MergeMetaData>& nodes, std::vector<MergeMetaData>& result) { - std::vector<uint16_t> idealNodes( - distribution.getIdealStorageNodes(state, bucketId, "ui")); - - // Add all ideal nodes first + // Add all ideal nodes first. These are never marked source-only. for (uint32_t i = 0; i < idealNodes.size(); i++) { - const MergeMetaData* entry = 0; + const MergeMetaData* entry = nullptr; for (uint32_t j = 0; j < nodes.size(); j++) { if (idealNodes[i] == nodes[j]._nodeIndex) { entry = &nodes[j]; @@ -39,52 +34,13 @@ MergeOperation::addIdealNodes( } } - if (entry != 0) { + if (entry != nullptr) { result.push_back(*entry); result.back()._sourceOnly = false; } } } -uint16_t -MergeOperation::countTrusted(const std::vector<MergeMetaData>& nodes) -{ - uint16_t trusted = 0; - for (const auto& n : nodes) { - if (n.trusted()) { - ++trusted; - } - } - return trusted; -} - -void -MergeOperation::addTrustedNodesNotAlreadyAdded( - uint16_t redundancy, - const std::vector<MergeMetaData>& nodes, - std::vector<MergeMetaData>& result) -{ - uint16_t alreadyTrusted = countTrusted(result); - for (uint32_t i = 0; i < nodes.size(); i++) { - if (!nodes[i].trusted()) { - continue; - } - - bool found = false; - for (uint32_t j = 0; j < result.size(); j++) { - if (result[j]._nodeIndex == nodes[i]._nodeIndex) { - found = true; - } - } - - if (!found) { - result.push_back(nodes[i]); - result.back()._sourceOnly = (alreadyTrusted >= redundancy); - ++alreadyTrusted; - } - } -} - void MergeOperation::addCopiesNotAlreadyAdded( uint16_t redundancy, @@ -114,11 +70,19 @@ MergeOperation::generateSortedNodeList( MergeLimiter& limiter, std::vector<MergeMetaData>& nodes) { + std::vector<uint16_t> idealNodes( + distribution.getIdealStorageNodes(state, bucketId, "ui")); + std::vector<MergeMetaData> result; const uint16_t redundancy = distribution.getRedundancy(); - addIdealNodes(distribution, state, bucketId, nodes, result); - addTrustedNodesNotAlreadyAdded(redundancy, nodes, result); + addIdealNodes(idealNodes, nodes, result); addCopiesNotAlreadyAdded(redundancy, nodes, result); + // TODO optimization: when merge case is obviously a replica move (all existing N replicas + // are in sync and new replicas are empty), could prune away N-1 lowest indexed replicas + // from the node list. This would minimize the number of nodes involved in the merge without + // sacrificing the end result. Avoiding the lower indexed nodes would take pressure off the + // merge throttling "locks" and could potentially greatly speed up node retirement in the common + // case. Existing replica could also be marked as source-only if it's not in the ideal state. limiter.limitMergeToMaxNodes(result); result.swap(nodes); } @@ -154,7 +118,7 @@ MergeOperation::onStart(DistributorMessageSender& sender) for (uint32_t i = 0; i < getNodes().size(); ++i) { const BucketCopy* copy = entry->getNode(getNodes()[i]); if (copy == 0) { // New copies? - newCopies.push_back(std::unique_ptr<BucketCopy>(new BucketCopy(0, getNodes()[i], api::BucketInfo()))); + newCopies.push_back(std::make_unique<BucketCopy>(0, getNodes()[i], api::BucketInfo())); copy = newCopies.back().get(); } nodes.push_back(MergeMetaData(getNodes()[i], *copy)); @@ -172,12 +136,11 @@ MergeOperation::onStart(DistributorMessageSender& sender) } if (_mnodes.size() > 1) { - std::shared_ptr<api::MergeBucketCommand> msg( - new api::MergeBucketCommand( - getBucketId(), - _mnodes, - _manager->getDistributorComponent().getUniqueTimestamp(), - clusterState.getVersion())); + auto msg = std::make_shared<api::MergeBucketCommand>( + getBucketId(), + _mnodes, + _manager->getDistributorComponent().getUniqueTimestamp(), + clusterState.getVersion()); // Due to merge forwarding/chaining semantics, we must always send // the merge command to the lowest indexed storage node involved in @@ -192,10 +155,7 @@ MergeOperation::onStart(DistributorMessageSender& sender) msg->setTimeout(60 * 60 * 1000); setCommandMeta(*msg); - sender.sendToNode( - lib::NodeType::STORAGE, - _mnodes[0].index, - msg); + sender.sendToNode(lib::NodeType::STORAGE, _mnodes[0].index, msg); _sentMessageTime = _manager->getDistributorComponent().getClock().getTimeInSeconds(); } else { @@ -213,13 +173,11 @@ MergeOperation::sourceOnlyCopyChangedDuringMerge( { assert(currentState.valid()); for (size_t i = 0; i < _mnodes.size(); ++i) { - const BucketCopy* copyBefore( - _infoBefore.getNode(_mnodes[i].index)); + const BucketCopy* copyBefore(_infoBefore.getNode(_mnodes[i].index)); if (!copyBefore) { continue; } - const BucketCopy* copyAfter( - currentState->getNode(_mnodes[i].index)); + const BucketCopy* copyAfter(currentState->getNode(_mnodes[i].index)); if (!copyAfter) { LOG(debug, "Copy of %s on node %u removed during merge. Was %s", getBucketId().toString().c_str(), @@ -295,6 +253,8 @@ MergeOperation::deleteSourceOnlyNodes( done(); } } + // FIXME what about the else-case here...? done() is not invoked by caller in this branch. + // Not calling done() doesn't leak anything, but causes metric updates to be missed. } void diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h index 8fdf22c88f5..f50226dbd3c 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h @@ -49,14 +49,7 @@ public: bool shouldBlockThisOperation(uint32_t messageType, uint8_t pri) const override ; private: static void addIdealNodes( - const lib::Distribution&, - const lib::ClusterState&, - const document::BucketId&, - const std::vector<MergeMetaData>& nodes, - std::vector<MergeMetaData>& result); - - static void addTrustedNodesNotAlreadyAdded( - uint16_t redundancy, + const std::vector<uint16_t>& idealNodes, const std::vector<MergeMetaData>& nodes, std::vector<MergeMetaData>& result); @@ -65,8 +58,6 @@ private: const std::vector<MergeMetaData>& nodes, std::vector<MergeMetaData>& result); - static uint16_t countTrusted(const std::vector<MergeMetaData>& nodes); - void deleteSourceOnlyNodes(const BucketDatabase::Entry& currentState, DistributorMessageSender& sender); }; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 7601d7faa6d..a6986c62715 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -5,7 +5,7 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.split"); diff --git a/storage/src/vespa/storage/distributor/operations/operation.cpp b/storage/src/vespa/storage/distributor/operations/operation.cpp index 7a1929115d3..8f6acc3258d 100644 --- a/storage/src/vespa/storage/distributor/operations/operation.cpp +++ b/storage/src/vespa/storage/distributor/operations/operation.cpp @@ -1,21 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* $Id$ */ - -#include <vespa/fastos/fastos.h> +#include "operation.h" #include <vespa/storage/common/distributorcomponent.h> -#include <vespa/storage/distributor/operations/operation.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/messageapi/storagereply.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> - LOG_SETUP(".distributor.callback"); -namespace storage { - -namespace distributor { +namespace storage::distributor { Operation::Operation() : _startTime(0) @@ -52,4 +47,3 @@ Operation::copyMessageSettings(const api::StorageCommand& source, api::StorageCo } -} diff --git a/storage/src/vespa/storage/distributor/operations/operation.h b/storage/src/vespa/storage/distributor/operations/operation.h index 50917334cfc..715ec88a2ba 100644 --- a/storage/src/vespa/storage/distributor/operations/operation.h +++ b/storage/src/vespa/storage/distributor/operations/operation.h @@ -4,14 +4,14 @@ #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/vdslib/state/nodetype.h> #include <vespa/storage/distributor/distributormessagesender.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/clock/time.h> -namespace storage -{ + +namespace storage { namespace api { -class StorageMessage; -class StorageReply; + class StorageMessage; + class StorageReply; } class StorageComponent; diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index fba6b131e09..6f331e23e2c 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -4,9 +4,9 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/printable.hpp> #include <sstream> +#include <cassert> -namespace storage { -namespace distributor { +namespace storage::distributor { BucketInstance::BucketInstance( const document::BucketId& id, const api::BucketInfo& info, @@ -197,5 +197,4 @@ OperationTargetResolverImpl::getAllInstances(OperationType type, return instances; } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index 9497bf05f17..5a2f0d35e8c 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -4,7 +4,7 @@ #include "bucketdbupdater.h" #include <vespa/storageframework/defaultimplementation/clock/realclock.h> #include <vespa/storage/common/bucketoperationlogger.h> -#include <vespa/vespalib/util/xmlserializable.hpp> +#include <vespa/vespalib/util/xmlstream.hpp> #include <vespa/log/log.h> LOG_SETUP(".pendingclusterstate"); diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index a234ab683b3..29f57000c18 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -4,7 +4,9 @@ #include "nodeinfo.h" #include "latency_statistics_provider.h" #include <vespa/storage/common/storagelink.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <vespa/storageframework/generic/component/componentregister.h> +#include <vespa/storageframework/generic/component/component.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vespalib/stllike/hash_set.h> diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index f174ecc65b8..8050f185583 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -4,6 +4,7 @@ #include "distributormetricsset.h" #include "distributorcomponent.h" #include "messagetracker.h" +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/messageapi/bucketinfocommand.h> #include <vespa/storageapi/messageapi/bucketinforeply.h> diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp index 0848de69981..02a6da98780 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp +++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/sentmessagemap.h> -#include <vespa/log/log.h> +#include "sentmessagemap.h" #include <vespa/storage/distributor/operations/operation.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".distributor.callback.map"); -namespace storage { - -namespace distributor { +namespace storage::distributor { SentMessageMap::SentMessageMap() : _map() @@ -92,5 +90,3 @@ SentMessageMap::clear() } } - -} diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index 096611e03e1..4d12318cb8f 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -1,9 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/statecheckers.h> -#include <vespa/log/log.h> -#include <vespa/storage/distributor/activecopy.h> +#include "statecheckers.h" +#include "activecopy.h" #include <vespa/storage/distributor/operations/idealstate/splitoperation.h> #include <vespa/storage/distributor/operations/idealstate/joinoperation.h> #include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> @@ -14,6 +12,7 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/log/log.h> LOG_SETUP(".distributor.operation.checkers"); namespace storage { diff --git a/storage/src/vespa/storage/distributor/statusreporterdelegate.cpp b/storage/src/vespa/storage/distributor/statusreporterdelegate.cpp index ecd880f2876..e2222d6fcdc 100644 --- a/storage/src/vespa/storage/distributor/statusreporterdelegate.cpp +++ b/storage/src/vespa/storage/distributor/statusreporterdelegate.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/statusreporterdelegate.h> +#include "statusreporterdelegate.h" -namespace storage { -namespace distributor { +namespace storage::distributor { StatusReporterDelegate::StatusReporterDelegate( framework::ComponentRegister& compReg, @@ -38,5 +36,5 @@ StatusReporterDelegate::registerStatusPage() _component.registerStatusPage(*this); } -} // distributor -} // storage +} + diff --git a/storage/src/vespa/storage/distributor/statusreporterdelegate.h b/storage/src/vespa/storage/distributor/statusreporterdelegate.h index 93a1fe1cd48..c8d17d227e7 100644 --- a/storage/src/vespa/storage/distributor/statusreporterdelegate.h +++ b/storage/src/vespa/storage/distributor/statusreporterdelegate.h @@ -3,7 +3,8 @@ #include "delegatedstatusrequest.h" #include "statusdelegator.h" -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/component/component.h> + namespace storage { namespace distributor { diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp index 883dc13cdb3..53967c1cae2 100644 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp +++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp @@ -6,9 +6,9 @@ #include <vespa/storage/storageutil/piechart.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <sstream> -#include <vespa/log/log.h> - +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".memory.status.viewer"); using storage::framework::defaultimplementation::MemoryState; diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h index dd7ecd7ae59..3ad378db78e 100644 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h +++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h @@ -36,7 +36,7 @@ #include <vespa/storage/common/storagecomponent.h> #include <vespa/storageframework/defaultimplementation/memory/memorystate.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> #include <deque> @@ -134,5 +134,4 @@ public: }; -} // storage - +} diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 18e0f2280f8..045a7514928 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -1,14 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statuswebserver.h" - +#include <vespa/storageframework/storageframework.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/fastlib/net/url.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/component/vtag.h> -#include <vespa/log/log.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".status"); namespace storage { @@ -23,14 +24,14 @@ StatusWebServer::StatusWebServer( _httpServer(), _configFetcher(configUri.getContext()), _queuedRequests(), - _component(componentRegister, "Status"), + _component(std::make_unique<framework::Component>(componentRegister, "Status")), _thread() { _configFetcher.subscribe<vespa::config::content::core::StorStatusConfig>(configUri.getConfigId(), this); _configFetcher.start(); framework::MilliSecTime maxProcessingTime(60 * 60 * 1000); framework::MilliSecTime maxWaitTime(10 * 1000); - _thread = _component.startThread(*this, maxProcessingTime, maxWaitTime); + _thread = _component->startThread(*this, maxProcessingTime, maxWaitTime); } @@ -40,8 +41,7 @@ StatusWebServer::~StatusWebServer() _configFetcher.close(); if (_httpServer.get() != 0) { - LOG(debug, "Shutting down status web server on port %u", - _httpServer->getListenPort()); + LOG(debug, "Shutting down status web server on port %u", _httpServer->getListenPort()); } // Delete http server to ensure that no more incoming requests reach us. _httpServer.reset(0); @@ -77,27 +77,23 @@ void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::St LOG(warning, "Listen failed on port %u", newPort); break; case FASTLIB_HTTPSERVER_NEWTHREADFAILED: - LOG(warning, "Failed starting thread for status server on " - "port %u", newPort); + LOG(warning, "Failed starting thread for status server on port %u", newPort); break; case FASTLIB_HTTPSERVER_ALREADYSTARTED: - LOG(warning, "Failed starting status server on port %u " - "(already started?)", newPort); + LOG(warning, "Failed starting status server on port %u (already started?)", newPort); break; default: - LOG(warning, "Failed starting status server on port %u " - "(unknown reason)", newPort); + LOG(warning, "Failed starting status server on port %u (unknown reason)", newPort); break; } if (!started) { std::ostringstream ost; - ost << "Failed to start status HTTP server using port " << newPort - << "."; + ost << "Failed to start status HTTP server using port " << newPort << "."; if (_httpServer.get() != 0) { - ost << " Status server still running on port " << _port - << " instead of suggested port " << newPort; + ost << " Status server still running on port " << _port << " instead of suggested port " << newPort; } - throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + LOG(fatal, "Failed to start status HTTP server using port %u. Old port was %u. Exiting.", newPort, _port); + std::quick_exit(67); } // Now that we know config update went well, update internal state _port = server->getListenPort(); @@ -152,8 +148,7 @@ namespace { } void -StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &serverSpec, - Fast_HTTPConnection& conn) +StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &serverSpec, Fast_HTTPConnection& conn) { Fast_URL urlCodec; int bufLength = tmpurl.length() * 2 + 10; @@ -182,37 +177,29 @@ StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &se // Route other status requests that can possibly deadlock to a // worker thread. vespalib::MonitorGuard monitor(_status._workerMonitor); - _status._queuedRequests.push_back( - HttpRequest::SP(new HttpRequest(url.c_str(), urlpath.getServerSpec()))); + _status._queuedRequests.emplace_back(std::make_shared<HttpRequest>(url.c_str(), urlpath.getServerSpec())); HttpRequest* req = _status._queuedRequests.back().get(); framework::SecondTime timeout(urlpath.get("timeout", 30u)); - framework::SecondTime timeoutTime( - _status._component.getClock().getTimeInSeconds() + timeout); + framework::SecondTime timeoutTime(_status._component->getClock().getTimeInSeconds() + timeout); monitor.signal(); while (true) { monitor.wait(100); bool done = false; if (req->_result.get()) { conn.Output(req->_result->c_str()); - LOG(debug, - "Finished status request for '%s'", - req->_url.c_str()); + LOG(debug, "Finished status request for '%s'", req->_url.c_str()); done = true; } else { - if (_status._component.getClock().getTimeInSeconds() - > timeoutTime) + if (_status._component->getClock().getTimeInSeconds() > timeoutTime) { std::ostringstream ost; { - HttpErrorWriter writer( - ost, "500 Internal Server Error"); + HttpErrorWriter writer(ost, "500 Internal Server Error"); writer << "Request " << url.c_str() << " timed out " << "after " << timeout << " seconds."; } - LOG(debug, - "HTTP status request failed: %s. %zu requests queued", - ost.str().c_str(), - _status._queuedRequests.size() - 1); + LOG(debug, "HTTP status request failed: %s. %zu requests queued", + ost.str().c_str(), _status._queuedRequests.size() - 1); conn.Output(ost.str().c_str()); done = true; } @@ -236,7 +223,7 @@ StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &se namespace { class IndexPageReporter : public framework::HtmlStatusReporter { std::ostringstream ost; - void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override{ + void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override { out << ost.str(); } @@ -249,8 +236,7 @@ namespace { } void -StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, - std::ostream& out) +StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, std::ostream& out) { vespalib::string link(urlpath.getPath()); if (link.size() > 0 && link[0] == '/') link = link.substr(1); @@ -258,29 +244,28 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, size_t slashPos = link.find('/'); if (slashPos != std::string::npos) link = link.substr(0, slashPos); - const framework::StatusReporter* reporter = 0; - if (link.size() > 0) { - reporter = _reporterMap.getStatusReporter(link); - } bool pageExisted = false; - if (reporter != 0) { - try{ - pageExisted = reporter->reportHttpHeader(out, urlpath); + if ( ! link.empty()) { + const framework::StatusReporter *reporter = _reporterMap.getStatusReporter(link); + if (reporter != nullptr) { + try { + pageExisted = reporter->reportHttpHeader(out, urlpath); + if (pageExisted) { + pageExisted = reporter->reportStatus(out, urlpath); + } + } catch (std::exception &e) { + HttpErrorWriter writer(out, "500 Internal Server Error"); + writer << "<pre>" << e.what() << "</pre>"; + pageExisted = true; + } catch (...) { + HttpErrorWriter writer(out, "500 Internal Server Error"); + writer << "Unknown exception"; + pageExisted = true; + } if (pageExisted) { - pageExisted = reporter->reportStatus(out, urlpath); + LOG(spam, "Status finished request"); + return; } - } catch (std::exception& e) { - HttpErrorWriter writer(out, "500 Internal Server Error"); - writer << "<pre>" << e.what() << "</pre>"; - pageExisted = true; - } catch (...) { - HttpErrorWriter writer(out, "500 Internal Server Error"); - writer << "Unknown exception"; - pageExisted = true; - } - if (pageExisted) { - LOG(spam, "Status finished request"); - return; } } if (!pageExisted && link.size() > 0) { @@ -291,11 +276,9 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, << vespalib::Vtag::currentVersion.toString() << "</p>\n"; { - std::vector<const framework::StatusReporter*> reporters( - _reporterMap.getStatusReporters()); - for (uint32_t i=0; i<reporters.size(); ++i) { - indexRep << "<a href=\"" << reporters[i]->getId() << "\">" - << reporters[i]->getName() << "</a><br>\n"; + for (const framework::StatusReporter * reporter : _reporterMap.getStatusReporters()) { + indexRep << "<a href=\"" << reporter->getId() << "\">" + << reporter->getName() << "</a><br>\n"; } } indexRep.reportHttpHeader(out, urlpath); @@ -322,12 +305,9 @@ StatusWebServer::run(framework::ThreadHandle& thread) HttpRequest::SP request; { vespalib::MonitorGuard monitor(_workerMonitor); - for (std::list<HttpRequest::SP>::iterator it - = _queuedRequests.begin(); it != _queuedRequests.end(); - ++it) - { - if ((*it)->_result.get() == 0) { - request = *it; + for (const HttpRequest::SP & cur : _queuedRequests) { + if ( ! cur->_result ) { + request = cur; break; } } @@ -340,8 +320,8 @@ StatusWebServer::run(framework::ThreadHandle& thread) framework::HttpUrlPath urlpath(request->_url, request->_serverSpec); std::ostringstream ost; handlePage(urlpath, ost); - // If the same request is still in front of the queue - // (it hasn't timed out), add the result to it. + // If the same request is still in front of the queue + // (it hasn't timed out), add the result to it. vespalib::MonitorGuard monitor(_workerMonitor); request->_result.reset(new vespalib::string(ost.str())); monitor.signal(); diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h index d1a0a33da95..f3abeb2d84a 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h @@ -11,7 +11,7 @@ #pragma once #include <vespa/storage/config/config-stor-status.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/fastlib/net/httpserver.h> @@ -19,6 +19,14 @@ namespace storage { +namespace framework { + class StatusReporterMap; + class ThreadHandle; + class ComponentRegister; + class Thread; + class HttpUrlPath; + class Component; +} class StatusWebServer : private config::IFetcherCallback<vespa::config::content::core::StorStatusConfig>, private framework::Runnable { @@ -29,9 +37,7 @@ class StatusWebServer : private config::IFetcherCallback<vespa::config::content: public: WebServer(StatusWebServer&, uint16_t port); - void onGetRequest(const string & url, - const string & serverSpec, - Fast_HTTPConnection& conn) override; + void onGetRequest(const string & url, const string & serverSpec, Fast_HTTPConnection& conn) override; const vespalib::string &getServerSpec() const { return _serverSpec; } @@ -50,14 +56,14 @@ class StatusWebServer : private config::IFetcherCallback<vespa::config::content: {} }; - framework::StatusReporterMap& _reporterMap; - vespalib::Monitor _workerMonitor; - uint16_t _port; - std::unique_ptr<WebServer> _httpServer; - config::ConfigFetcher _configFetcher; - std::list<HttpRequest::SP> _queuedRequests; - framework::Component _component; - framework::Thread::UP _thread; + framework::StatusReporterMap& _reporterMap; + vespalib::Monitor _workerMonitor; + uint16_t _port; + std::unique_ptr<WebServer> _httpServer; + config::ConfigFetcher _configFetcher; + std::list<HttpRequest::SP> _queuedRequests; + std::unique_ptr<framework::Component> _component; + std::unique_ptr<framework::Thread> _thread; public: StatusWebServer(const StatusWebServer &) = delete; @@ -65,7 +71,7 @@ public: StatusWebServer(framework::ComponentRegister&, framework::StatusReporterMap&, const config::ConfigUri & configUri); - virtual ~StatusWebServer(); + ~StatusWebServer() override; void handlePage(const framework::HttpUrlPath&, std::ostream& out); static vespalib::string getServerSpec(const vespalib::string &requestSpec, diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 2861ab904ff..3d9586ea398 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -5,7 +5,7 @@ #include <vespa/storage/bucketmover/htmltable.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".deadlock.detector"); namespace storage { diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h index c8cb3f35ddf..34a345330b7 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h @@ -15,7 +15,8 @@ #include "appkiller.h" #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> +#include <vespa/storageframework/generic/thread/threadpool.h> #include <vespa/vespalib/util/sync.h> #include <map> #include <atomic> diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index b26c6ec893f..0a173b2945c 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -7,7 +7,7 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/backtrace.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.bucketownershipnotifier"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/diskthread.h b/storage/src/vespa/storage/persistence/diskthread.h index a1af83a65ef..c2982de3236 100644 --- a/storage/src/vespa/storage/persistence/diskthread.h +++ b/storage/src/vespa/storage/persistence/diskthread.h @@ -14,10 +14,12 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/util/document_runnable.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/config-stor-filestor.h> +#include <vespa/storageframework/generic/thread/runnable.h> + namespace storage { + namespace framework { class Thread; } diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.cpp b/storage/src/vespa/storage/persistence/fieldvisitor.cpp index e67d15d4e91..6657a73543e 100644 --- a/storage/src/vespa/storage/persistence/fieldvisitor.cpp +++ b/storage/src/vespa/storage/persistence/fieldvisitor.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // @author Vegard Sjonfjell -#include <vespa/storage/persistence/fieldvisitor.h> + +#include "fieldvisitor.h" +#include <vespa/document/select/valuenodes.h> namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index b549aca35b6..41572042e35 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -29,6 +29,9 @@ namespace api { namespace spi { class PartitionStateList; } +namespace framework { + class HttpUrlPath; +} class FileStorHandlerImpl; class FileStorMetrics; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index dc2bb96d67b..a530b59d4d9 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -20,7 +20,7 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/metrics/metrics.h> #include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index a3c8fc3d671..128eff61436 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -17,7 +17,9 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.filestor.manager"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 39d9a60a950..38c735ac767 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -24,8 +24,9 @@ #include <vespa/storage/persistence/diskthread.h> #include <vespa/storage/persistence/providershutdownwrapper.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> + #include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/ifetchercallback.h> #include <vespa/config/config.h> diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h index c5d2dcae65c..a395ec08b12 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h @@ -2,10 +2,10 @@ #pragma once #include <vespa/persistence/spi/context.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucket.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vector> #include <deque> diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 04c8c7b2391..9938d6522e7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <memory> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/types.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/document/bucket/bucketidlist.h> #include <vespa/vespalib/util/sync.h> #include <vespa/config/config.h> @@ -52,8 +51,7 @@ private: } bool requestModifiedBucketsFromProvider(); void nextRecheckChunk(std::vector<RecheckBucketInfoCommand::SP>&); - void dispatchAllToPersistenceQueues( - const std::vector<RecheckBucketInfoCommand::SP>&); + void dispatchAllToPersistenceQueues(const std::vector<RecheckBucketInfoCommand::SP>&); spi::PersistenceProvider& _provider; ServiceLayerComponent::UP _component; @@ -61,7 +59,7 @@ private: config::ConfigFetcher _configFetcher; vespalib::Monitor _monitor; vespalib::Lock _stateLock; - document::BucketId::List _rechecksNotStarted; + document::bucket::BucketIdList _rechecksNotStarted; size_t _pendingRequests; size_t _maxPendingChunkSize; bool _singleThreadMode; // For unit testing only diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 10a5a7505dd..011c6ecc1b0 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -11,7 +11,8 @@ #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> + +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.thread"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index d2c4dba3a5f..ed9cce8a54c 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -8,7 +8,6 @@ #include "diskmoveoperationhandler.h" #include "persistenceutil.h" #include "providershutdownwrapper.h" -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/statusmessages.h> diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 2d8a8561ba6..202a7c5cac5 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -4,7 +4,7 @@ #include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.util"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index c917d4d0149..d06afb664f7 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -7,7 +7,6 @@ #include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/storage/persistence/filestorage/pausehandler.h> #include <vespa/storage/persistence/types.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/storage/storageutil/utils.h> #include <vespa/config-stor-filestor.h> diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index 62039f36133..cf3fde77b2b 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/persistence/splitbitdetector.h> -#include <vespa/storage/persistence/bucketprocessor.h> +#include "splitbitdetector.h" +#include "bucketprocessor.h" #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.split.bitdetector"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/types.h b/storage/src/vespa/storage/persistence/types.h index de31145f27b..234c3adfc02 100644 --- a/storage/src/vespa/storage/persistence/types.h +++ b/storage/src/vespa/storage/persistence/types.h @@ -9,8 +9,8 @@ #include <vespa/storageapi/buckets/bucketinfo.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/defs.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/storageframework/generic/clock/time.h> namespace storage { diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 598d84b934d..c285645309a 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/bouncer.h> - -#include <vespa/log/log.h> +#include "bouncer.h" #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/persistence.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".bouncer"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp index 14b6adcb9a4..05b581d839a 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp @@ -7,10 +7,10 @@ #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/config/config.h> #include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketintegritychecker"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h index fd9516a85aa..f55cb18366c 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h @@ -14,8 +14,9 @@ #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/storage/config/config-stor-integritychecker.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/config/config.h> +#include <list> namespace storage { diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 27261424894..3390afcb9ec 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -7,7 +7,7 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".bucketownershiphandler"); namespace storage { @@ -56,8 +56,7 @@ void ChangedBucketOwnershipHandler::reloadClusterState() { vespalib::LockGuard guard(_stateLock); - lib::ClusterState::CSP newState(_component.getStateUpdater() - .getSystemState()); + lib::ClusterState::CSP newState(_component.getStateUpdater().getSystemState()); setCurrentOwnershipWithStateNoLock(*newState); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index c2b9a438967..f32b1c242cf 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -2,6 +2,7 @@ #include "communicationmanager.h" #include "fnetlistener.h" #include "rpcrequestwrapper.h" +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/storageapi/message/state.h> #include <vespa/messagebus/emptyreply.h> @@ -9,8 +10,8 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".communication.manager"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index ab7ffcf936c..c5d17294dd7 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -16,7 +16,7 @@ #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/config/config-stor-communicationmanager.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageapi/mbusprot/storagecommand.h> #include <vespa/storageapi/mbusprot/storagereply.h> #include <vespa/messagebus/rpcmessagebus.h> diff --git a/storage/src/vespa/storage/storageserver/distributornodecontext.cpp b/storage/src/vespa/storage/storageserver/distributornodecontext.cpp index f44f7ecbbf2..b378ca8efa4 100644 --- a/storage/src/vespa/storage/storageserver/distributornodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/distributornodecontext.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/distributornodecontext.h> +#include "distributornodecontext.h" namespace storage { diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index c2965bcddd5..ae789891852 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/documentapiconverter.h> +#include "documentapiconverter.h" #include <vespa/documentapi/documentapi.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/datagram.h> @@ -14,10 +13,10 @@ #include <vespa/storageapi/message/batch.h> #include <vespa/messagebus/errorcode.h> #include <vespa/storageapi/messageapi/returncode.h> -#include <vespa/log/log.h> #include <vespa/vdslib/container/documentlist.h> #include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/log/log.h> LOG_SETUP(".documentapiconverter"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.cpp b/storage/src/vespa/storage/storageserver/fnetlistener.cpp index 2c557ba8614..7daf2fb4777 100644 --- a/storage/src/vespa/storage/storageserver/fnetlistener.cpp +++ b/storage/src/vespa/storage/storageserver/fnetlistener.cpp @@ -5,6 +5,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/host_name.h> +#include <sstream> #include <vespa/log/log.h> @@ -14,17 +15,17 @@ namespace storage { FNetListener::FNetListener(CommunicationManager& comManager, const config::ConfigUri & configUri, uint32_t port) : _comManager(comManager), - _orb(), + _orb(std::make_unique<FRT_Supervisor>()), _closed(false), - _slobrokRegister(_orb, configUri) + _slobrokRegister(*_orb, configUri) { initRPC(); - if (!_orb.Listen(port)) { + if (!_orb->Listen(port)) { std::ostringstream ost; ost << "Failed to listen to RPC port " << port << "."; throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); } - _orb.Start(); + _orb->Start(); } FNetListener::~FNetListener() @@ -49,13 +50,13 @@ FNetListener::close() { _closed = true; _slobrokRegister.unregisterName(_handle); - _orb.ShutDown(true); + _orb->ShutDown(true); } void FNetListener::initRPC() { - FRT_ReflectionBuilder rb(&_orb); + FRT_ReflectionBuilder rb(_orb.get()); rb.DefineMethod( "getnodestate3", "sii", "ss", true, diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.h b/storage/src/vespa/storage/storageserver/fnetlistener.h index 40cd53eaa63..781f818b673 100644 --- a/storage/src/vespa/storage/storageserver/fnetlistener.h +++ b/storage/src/vespa/storage/storageserver/fnetlistener.h @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> -#include <vespa/slobrok/sbmirror.h> #include <vespa/slobrok/sbregister.h> - namespace storage { class CommunicationManager; @@ -26,17 +23,15 @@ public: void registerHandle(const vespalib::stringref & handle); void close(); - // Used by unit tests. bool serviceExists(const vespalib::stringref & connectionSpec); private: - CommunicationManager& _comManager; - FRT_Supervisor _orb; - bool _closed; + CommunicationManager& _comManager; + std::unique_ptr<FRT_Supervisor> _orb; + bool _closed; slobrok::api::RegisterAPI _slobrokRegister; - vespalib::string _handle; + vespalib::string _handle; }; } - diff --git a/storage/src/vespa/storage/storageserver/framework.cpp b/storage/src/vespa/storage/storageserver/framework.cpp index 1bdd3ad9e04..781f5cb0e77 100644 --- a/storage/src/vespa/storage/storageserver/framework.cpp +++ b/storage/src/vespa/storage/storageserver/framework.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/framework.h> +#include "framework.h" #include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 577fedb58e1..ede7be3b9ad 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -2,9 +2,9 @@ #include "mergethrottler.h" #include "storagemetricsset.h" -#include <iostream> #include <sstream> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/persistence/messages.h> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 03662cfadee..3dca5f6dbe0 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -10,8 +10,8 @@ #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/distributor/messageguard.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/document_runnable.h> diff --git a/storage/src/vespa/storage/storageserver/messagedispatcher.cpp b/storage/src/vespa/storage/storageserver/messagedispatcher.cpp index a35b2c4157d..54beb2a7ad3 100644 --- a/storage/src/vespa/storage/storageserver/messagedispatcher.cpp +++ b/storage/src/vespa/storage/storageserver/messagedispatcher.cpp @@ -1,16 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* $Id$ */ +#include "messagedispatcher.h" -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/messagedispatcher.h> - -#include <vespa/log/log.h> -#include <vespa/document/bucket/bucketid.h> #include <vespa/storageapi/message/state.h> #include <storageapi/messageapi/chainedcommand.h> #include <storageapi/messageapi/chainedreply.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/log/log.h> LOG_SETUP(".message.dispatcher"); using std::shared_ptr; diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp index 0df24ef3536..4960cd2085e 100644 --- a/storage/src/vespa/storage/storageserver/messagesink.cpp +++ b/storage/src/vespa/storage/storageserver/messagesink.cpp @@ -2,6 +2,7 @@ #include "messagesink.h" #include <vespa/storageapi/message/persistence.h> +#include <ostream> using std::shared_ptr; diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index b5d66c837c5..f770fc0af51 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/opslogger.h> - -#include <vespa/log/log.h> +#include "opslogger.h" #include <vespa/storageapi/message/persistence.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".operationslogger"); namespace storage { diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.cpp b/storage/src/vespa/storage/storageserver/priorityconverter.cpp index ff4f2c086ce..c8cf9e5fc29 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.cpp +++ b/storage/src/vespa/storage/storageserver/priorityconverter.cpp @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/documentapi/messagebus/documentprotocol.h> + #include "priorityconverter.h" +#include <vespa/documentapi/messagebus/documentprotocol.h> namespace storage { diff --git a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp index 46d518b90c3..cd60b2739bf 100644 --- a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp +++ b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".api.rpc.request"); + #include "rpcrequestwrapper.h" +#include <vespa/fnet/frt/rpcrequest.h> +#include <cassert> namespace storage { @@ -14,8 +14,7 @@ RPCRequestWrapper::RPCRequestWrapper(FRT_RPCRequest *req) RPCRequestWrapper::~RPCRequestWrapper() { if (_req != 0) { - _req->SetError(ERR_REQUEST_DELETED, - "Request deleted without having been replied to"); + _req->SetError(ERR_REQUEST_DELETED, "Request deleted without having been replied to"); _req->Return(); } } @@ -82,6 +81,11 @@ RPCRequestWrapper::returnRequest() } +const char * +RPCRequestWrapper::getMethodName() const { + return _req->GetMethodName(); +} + void RPCRequestWrapper::discardBlobs() { diff --git a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h index a1e067ca974..fdb685cab27 100644 --- a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h +++ b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h @@ -1,7 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fnet/frt/frt.h> +#include <cstdint> + +class FRT_RPCRequest; namespace storage { @@ -47,7 +49,7 @@ public: **/ void returnError(uint32_t errorCode, const char *errorMessage); - const char *getMethodName() { return _req->GetMethodName(); } + const char *getMethodName() const; void addReturnString(const char *str, uint32_t len=0); void addReturnInt(uint32_t value); void returnRequest(); @@ -67,4 +69,3 @@ private: }; } // namespace storage - diff --git a/storage/src/vespa/storage/storageserver/servicelayernodecontext.cpp b/storage/src/vespa/storage/storageserver/servicelayernodecontext.cpp index afdab9b6bce..87b8c62024c 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernodecontext.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/servicelayernodecontext.h> +#include "servicelayernodecontext.h" namespace storage { diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index bfe0418c929..703ebbc69c1 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -17,13 +17,14 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/objects/floatingpointtype.h> #include <deque> #include <map> +#include <list> #include <atomic> namespace metrics { diff --git a/storage/src/vespa/storage/storageserver/statereporter.h b/storage/src/vespa/storage/storageserver/statereporter.h index c46a878ef40..98a29c51d8a 100644 --- a/storage/src/vespa/storage/storageserver/statereporter.h +++ b/storage/src/vespa/storage/storageserver/statereporter.h @@ -12,7 +12,7 @@ #include "applicationgenerationfetcher.h" #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/status/statusreporter.h> #include <vespa/metrics/metrics.h> #include <vespa/metrics/state_api_adapter.h> #include <vespa/vespalib/net/metrics_producer.h> diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 5eea62a17ad..cf47b99164f 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -35,8 +35,8 @@ #include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storage/visiting/visitormessagesessionfactory.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/config-upgrading.h> #include <vespa/config-stor-distribution.h> diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp index 90aefcaceae..c47ed97f879 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageserver/storagenodecontext.h> +#include "storagenodecontext.h" #include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> diff --git a/storage/src/vespa/storage/storageutil/bloomfilter.cpp b/storage/src/vespa/storage/storageutil/bloomfilter.cpp index 26e9e4012d7..87498aca0d2 100644 --- a/storage/src/vespa/storage/storageutil/bloomfilter.cpp +++ b/storage/src/vespa/storage/storageutil/bloomfilter.cpp @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "bloomfilter.h" -#include <stdlib.h> -#include <string.h> +#include <cstdlib> +#include <cstring> BloomFilter::BloomFilter(int size, int hashes, uint32_t *buf) : _size(size), diff --git a/storage/src/vespa/storage/storageutil/palette.cpp b/storage/src/vespa/storage/storageutil/palette.cpp index 76998bd7677..f6a5c13e5b3 100644 --- a/storage/src/vespa/storage/storageutil/palette.cpp +++ b/storage/src/vespa/storage/storageutil/palette.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/storageutil/palette.h> +#include "palette.h" #include <iostream> #include <iomanip> diff --git a/storage/src/vespa/storage/tools/.gitignore b/storage/src/vespa/storage/tools/.gitignore index 6f27b7c441d..ba1c4352573 100644 --- a/storage/src/vespa/storage/tools/.gitignore +++ b/storage/src/vespa/storage/tools/.gitignore @@ -16,7 +16,7 @@ populatenode slotfilefeeder statfs stoccart -storage-cmd +vespa-storage-cmd throttlingsim vdsclient vdsdisktool diff --git a/storage/src/vespa/storage/tools/CMakeLists.txt b/storage/src/vespa/storage/tools/CMakeLists.txt index 3f5febcd28a..f2bbb266ef9 100644 --- a/storage/src/vespa/storage/tools/CMakeLists.txt +++ b/storage/src/vespa/storage/tools/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_executable(storage_getidealstate_app SOURCES getidealstate.cpp - INSTALL bin DEPENDS storage AFTER @@ -11,7 +10,6 @@ vespa_add_executable(storage_getidealstate_app vespa_add_executable(storage_generatedistributionbits_app SOURCES generatedistributionbits.cpp - INSTALL bin DEPENDS storage AFTER @@ -20,7 +18,6 @@ vespa_add_executable(storage_generatedistributionbits_app vespa_add_executable(storage_analyzedistribution_app SOURCES analyzedistribution.cpp - INSTALL bin DEPENDS storage AFTER @@ -29,8 +26,7 @@ vespa_add_executable(storage_analyzedistribution_app vespa_add_executable(storage_storage-cmd_app SOURCES storage-cmd.cpp - OUTPUT_NAME storage-cmd - INSTALL bin + OUTPUT_NAME vespa-storage-cmd DEPENDS AFTER storage_storageconfig @@ -38,7 +34,6 @@ vespa_add_executable(storage_storage-cmd_app vespa_add_executable(storage_throttlingsim_app SOURCES throttlingsim.cpp - INSTALL bin DEPENDS AFTER storage_storageconfig @@ -46,7 +41,6 @@ vespa_add_executable(storage_throttlingsim_app vespa_add_executable(storage_statfs_app SOURCES statfs.cpp - INSTALL bin DEPENDS AFTER storage_storageconfig diff --git a/storage/src/vespa/storage/tools/analyzedistribution.cpp b/storage/src/vespa/storage/tools/analyzedistribution.cpp index 0368e46c668..a1bddac8f37 100644 --- a/storage/src/vespa/storage/tools/analyzedistribution.cpp +++ b/storage/src/vespa/storage/tools/analyzedistribution.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketdb/judyarray.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/programoptions.h> @@ -8,7 +8,6 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/state/nodestate.h> -#include <vespa/storage/bucketdb/judyarray.h> #include <iostream> #include <sstream> diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp index 53c7c7cf1a1..3f5b3f4ae3d 100644 --- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp +++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp @@ -1,9 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vespalib/util/programoptions.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/bucketdb/judyarray.h> +#include <vespa/vespalib/util/stringfmt.h> #include <iomanip> #include <iostream> #include <vespa/config-stor-distribution.h> diff --git a/storage/src/vespa/storage/tools/storage-cmd.cpp b/storage/src/vespa/storage/tools/storage-cmd.cpp index 8b0496507c5..b9ead2be834 100644 --- a/storage/src/vespa/storage/tools/storage-cmd.cpp +++ b/storage/src/vespa/storage/tools/storage-cmd.cpp @@ -1,9 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("rpc_invoke"); #include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/sbmirror.h> +#include <vespa/fastos/app.h> + +#include <vespa/log/log.h> +LOG_SETUP("vespa-storage-cmd"); class RPCClient : public FastOS_Application { @@ -45,7 +46,7 @@ private: public: int Main() override { if (_argc < 3) { - fprintf(stderr, "usage: storage-cmd <connectspec> <method> [args]\n"); + fprintf(stderr, "usage: vespa-storage-cmd <connectspec> <method> [args]\n"); fprintf(stderr, "Calls RPC method on a storage/distributor process\n"); fprintf(stderr, "Call frt.rpc.getMethodList to get available RPC methods\n"); fprintf(stderr, " each arg must be on the form <type>:<value>\n"); diff --git a/storage/src/vespa/storage/tools/throttlingsim.cpp b/storage/src/vespa/storage/tools/throttlingsim.cpp index cd9e5a29d81..9d45df9b2e5 100644 --- a/storage/src/vespa/storage/tools/throttlingsim.cpp +++ b/storage/src/vespa/storage/tools/throttlingsim.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "throttlingsim.h" #include <algorithm> #include <vespa/vespalib/util/stringfmt.h> diff --git a/storage/src/vespa/storage/tools/throttlingsim.h b/storage/src/vespa/storage/tools/throttlingsim.h index a7384d0f362..55fccb01acd 100644 --- a/storage/src/vespa/storage/tools/throttlingsim.h +++ b/storage/src/vespa/storage/tools/throttlingsim.h @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/util/document_runnable.h> +#include <vespa/fastos/app.h> #include <deque> #include <vector> diff --git a/storage/src/vespa/storage/visiting/commandqueue.h b/storage/src/vespa/storage/visiting/commandqueue.h index ba0141add45..070cc72e32f 100644 --- a/storage/src/vespa/storage/visiting/commandqueue.h +++ b/storage/src/vespa/storage/visiting/commandqueue.h @@ -17,8 +17,8 @@ #include <boost/multi_index/sequenced_index.hpp> #include <vespa/vespalib/util/printable.h> #include <vespa/fastos/timestamp.h> +#include <vespa/storageframework/generic/clock/clock.h> #include <list> -#include <vespa/storageframework/storageframework.h> namespace storage { diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp index 282e2f32873..ca6678c1b3d 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.cpp +++ b/storage/src/vespa/storage/visiting/countvisitor.cpp @@ -3,6 +3,7 @@ #include "countvisitor.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.countvisitor"); diff --git a/storage/src/vespa/storage/visiting/dumpvisitor.cpp b/storage/src/vespa/storage/visiting/dumpvisitor.cpp index 388f9dc5e19..3eeee84e848 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitor.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitor.cpp @@ -4,6 +4,7 @@ #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vdslib/container/mutabledocumentlist.h> +#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.dumpvisitor"); diff --git a/storage/src/vespa/storage/visiting/messages.h b/storage/src/vespa/storage/visiting/messages.h index b79fae0575f..d5b3c501aba 100644 --- a/storage/src/vespa/storage/visiting/messages.h +++ b/storage/src/vespa/storage/visiting/messages.h @@ -9,7 +9,6 @@ #include <vespa/storageapi/message/internal.h> #include <vespa/storage/visiting/config-stor-visitor.h> -#include <vespa/storageframework/storageframework.h> namespace storage { diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 9ae18d7b05d..7fbe87cecad 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -4,6 +4,7 @@ #include "recoveryvisitor.h" #include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 30bf3fe9757..aff3cb0725a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,13 +2,18 @@ #include "visitor.h" #include "visitormetrics.h" +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/document/select/node.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <unordered_map> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".visitor.instance"); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 918d2d0e339..448de6d1381 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -22,34 +22,31 @@ #include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> -#include <vespa/storageframework/storageframework.h> #include <list> #include <deque> namespace document { -class Document; -class DocumentId; -namespace select { -class Node; -} -} -namespace vdslib { -class Parameters; + class Document; + class DocumentId; + namespace select { class Node; } } +namespace vdslib { class Parameters; } namespace documentapi { -class DocumentMessage; -class VisitorInfoMessage; + class DocumentMessage; + class VisitorInfoMessage; } namespace storage { namespace api { -class ReturnCode; -class StorageCommand; -class StorageReply; + class ReturnCode; + class StorageCommand; + class StorageReply; } +namespace framework { class MemoryAllocationType; } + class GetIterReply; class CreateIteratorReply; class Visitor; @@ -62,8 +59,7 @@ class VisitorThreadMetrics; */ class VisitorMessageHandler { public: - virtual void send(const std::shared_ptr<api::StorageCommand>&, - Visitor& visitor) = 0; + virtual void send(const std::shared_ptr<api::StorageCommand>&, Visitor& visitor) = 0; virtual void send(const std::shared_ptr<api::StorageReply>&) = 0; /** * Called once when visitor shuts down and won't call this handler again. @@ -602,4 +598,3 @@ private: } // storage - diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 33beae14c4c..5e1f8b4df79 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -7,10 +7,11 @@ #include "countvisitor.h" #include "testvisitor.h" #include "recoveryvisitor.h" -#include <vespa/storageapi/message/visitor.h> +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/storage/common/statusmessages.h> -#include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".visitor.manager"); diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 119517d8d2a..5d7ec0caf1b 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -24,8 +24,8 @@ #include "visitormetrics.h" #include "visitorthread.h" #include <vespa/storage/visiting/config-stor-visitor.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/storagelink.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> @@ -33,9 +33,9 @@ #include <vespa/vespalib/util/document_runnable.h> namespace storage { -namespace api { - class BucketTimeInterval; -} + +namespace api { class BucketTimeInterval; } + class RequestStatusPageReply; class VisitorManager : public framework::Runnable, diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 4b96faee14b..aeea3c7ba8e 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -2,6 +2,7 @@ #include "visitorthread.h" #include "messages.h" +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/orderingselector.h> @@ -13,8 +14,10 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/base/exceptions.h> -#include <locale> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <locale> +#include <sstream> + #include <vespa/log/log.h> LOG_SETUP(".visitor.thread"); diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 5f9d8897d9f..c5debcccafc 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -16,9 +16,9 @@ #include "visitor.h" #include "visitormetrics.h" #include "visitormessagesessionfactory.h" -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storage/common/storagecomponent.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/metrics/metrictimer.h> #include <vespa/vespalib/util/document_runnable.h> @@ -27,6 +27,8 @@ namespace storage { +namespace framework { class HttpUrlPath; } + class VisitorThread : public framework::Runnable, private api::MessageHandler, private framework::MetricUpdateHook diff --git a/storageapi/OWNERS b/storageapi/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/storageapi/OWNERS +++ b/storageapi/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp index 07ba2773e91..f9924c953c2 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/xmlstream.h> -namespace storage { -namespace api { +namespace storage::api { BucketInfo::BucketInfo() : _lastModified(0), @@ -124,5 +124,4 @@ BucketInfo::printXml(vespalib::XmlOutputStream& xos) const << XmlAttribute("lastmodified", _lastModified); } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 1ad07387ba4..9d4d25d9d8e 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -13,14 +13,13 @@ #include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/vespalib/util/exceptions.h> - #include <vespa/vespalib/util/growablebytebuffer.h> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".storage.api.mbusprot.serialization.base"); -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { ProtocolSerialization::ProtocolSerialization( const document::DocumentTypeRepo::SP& repo) @@ -299,5 +298,4 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, return StorageReply::UP(new StorageReply(SRep::SP(reply.release()))); } -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 62a85e0f45e..87dcacb0fb9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -5,12 +5,12 @@ #include "storagereply.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/document/util/stringutil.h> +#include <sstream> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.api.mbusprot.protocol"); -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { mbus::string StorageProtocol::NAME = "StorageProtocol"; @@ -169,5 +169,4 @@ StorageProtocol::decode(const vespalib::Version & version, return mbus::Routable::UP(); } -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp index 405d5b4c9d2..ee7bbbcbdf6 100644 --- a/storageapi/src/vespa/storageapi/message/batch.cpp +++ b/storageapi/src/vespa/storageapi/message/batch.cpp @@ -2,7 +2,7 @@ // #include "batch.h" #include <vespa/document/bucket/bucketidfactory.h> -#include <algorithm> +#include <ostream> using namespace storage::api; diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index 3400e0d6399..e4e6f796ac2 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -4,9 +4,10 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/array.hpp> +#include <ostream> +#include <iterator> -namespace storage { -namespace api { +namespace storage::api { IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply) IMPLEMENT_REPLY(CreateBucketReply) @@ -625,7 +626,6 @@ SetBucketStateReply::print(std::ostream& out, } } -} // api -} // storage +} template class vespalib::Array<storage::api::RequestBucketInfoReply::Entry>; diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp index 6de53d9db82..ce1adcfa2ca 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -3,6 +3,7 @@ #include "persistence.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <ostream> namespace storage { namespace api { diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 536135c48e6..399a30543da 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -2,13 +2,29 @@ #include "storagemessage.h" -#include <vespa/vespalib/util/exceptions.h> #include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/asciistream.h> #include <sstream> +#include <cassert> + +namespace storage::api { + +namespace { -namespace storage { -namespace api { +/** + * TODO + * From @vekterli + * I have no idea why the _lastMsgId update code masks away the 8 MSB, but if we assume it's probably for no + * overwhelmingly good reason we could replace this mutex with just a std::atomic<uint64_t> and do a relaxed + * fetch_add (shouldn't be any need for any barriers; ID increments have no other memory dependencies). U64 overflows + * here come under the category "never gonna happen in the real world". + * @balder agree - @vekterli fix in separate pull request :) + */ +vespalib::Lock _G_msgIdLock; + +} static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster."; @@ -177,6 +193,20 @@ MessageType::MessageType::get(Id id) } return *it->second; } +MessageType::MessageType(const vespalib::stringref & name, Id id, + const MessageType* replyOf) + : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) +{ + _codes[id] = this; + if (_replyOf != 0) { + assert(_replyOf->_reply == 0); + // Ugly cast to let initialization work + MessageType& type = const_cast<MessageType&>(*_replyOf); + type._reply = this; + } +} + +MessageType::~MessageType() {} void MessageType::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -305,17 +335,14 @@ StorageMessageAddress::print(vespalib::asciistream & out) const } } -TransportContext::~TransportContext() -{ -} +TransportContext::~TransportContext() { } -vespalib::Lock StorageMessage::_msgIdLock; StorageMessage::Id StorageMessage::_lastMsgId = 1000; StorageMessage::Id StorageMessage::generateMsgId() { - vespalib::LockGuard sync(_msgIdLock); + vespalib::LockGuard sync(_G_msgIdLock); Id msgId = _lastMsgId++; _lastMsgId &= ((Id(-1) << 8) >> 8); return msgId; @@ -343,7 +370,7 @@ StorageMessage::~StorageMessage() { } void StorageMessage::setNewMsgId() { - vespalib::LockGuard sync(_msgIdLock); + vespalib::LockGuard sync(_G_msgIdLock); _msgId = _lastMsgId++; _lastMsgId &= ((Id(-1) << 8) >> 8); } @@ -353,5 +380,5 @@ StorageMessage::getSummary() const { return toString(); } -} // api -} // storage +} + diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index 59ae462432b..442ee0c9899 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -19,7 +19,6 @@ #include <vespa/vdslib/state/nodetype.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/util/sync.h> #include <map> namespace vespalib { @@ -82,10 +81,6 @@ namespace api { * This is used to be able to deserialize messages of various classes. */ class MessageType : public vespalib::Printable { -private: - MessageType(const MessageType &); - MessageType& operator=(const MessageType &); - public: enum Id { GET_ID = 4, @@ -172,18 +167,7 @@ private: MessageType *_reply; const MessageType *_replyOf; - MessageType(const vespalib::stringref & name, Id id, - const MessageType* replyOf = 0) - : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) - { - _codes[id] = this; - if (_replyOf != 0) { - assert(_replyOf->_reply == 0); - // Ugly cast to let initialization work - MessageType& type = const_cast<MessageType&>(*_replyOf); - type._reply = this; - } - } + MessageType(const vespalib::stringref & name, Id id, const MessageType* replyOf = 0); public: static const MessageType DOCBLOCK; static const MessageType DOCBLOCK_REPLY; @@ -264,16 +248,17 @@ public: static const MessageType& get(Id id); + MessageType(const MessageType &) = delete; + MessageType& operator=(const MessageType &) = delete; + ~MessageType(); Id getId() const { return _id; } static Id getMaxId() { return MESSAGETYPE_MAX_ID; } const vespalib::string& getName() const { return _name; } bool isReply() const { return (_replyOf != 0); } - /** Only valid to call on replies. */ - const MessageType& getCommandType() const - { assert(_replyOf); return *_replyOf; } - /** Only valid to call on commands. */ - const MessageType& getReplyType() const - { assert(_reply); return *_reply; } + /** Only valid to call on replies. */ + const MessageType& getCommandType() const { return *_replyOf; } + /** Only valid to call on commands. */ + const MessageType& getReplyType() const { return *_reply; } bool operator==(const MessageType& type) const { return (_id == type._id); } bool operator!=(const MessageType& type) const { return (_id != type._id); } @@ -294,7 +279,7 @@ private: mbus::Route _route; bool _retryEnabled; Protocol _protocol; - // Used for internal VDS addresses only + // Used for internal VDS addresses only vespalib::string _cluster; const lib::NodeType* _type; uint16_t _index; @@ -346,7 +331,6 @@ public: static const char* getPriorityString(Priority); private: - static vespalib::Lock _msgIdLock; static Id _lastMsgId; StorageMessage& operator=(const StorageMessage&); diff --git a/storageframework/OWNERS b/storageframework/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/storageframework/OWNERS +++ b/storageframework/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h b/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h index c4659bea21e..ae2ebda5fd3 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h @@ -8,7 +8,6 @@ #pragma once #include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/sync.h> namespace storage { diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h b/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h index a8fd5d597d7..a4c38d24fbd 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h @@ -9,11 +9,9 @@ */ #pragma once -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/clock/clock.h> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { struct RealClock : public Clock { MicroSecTime getTimeInMicros() const override; @@ -22,7 +20,4 @@ struct RealClock : public Clock { MonotonicTimePoint getMonotonicTime() const override; }; -} // defaultimplementation -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h index 41fb819ae68..247970f62ef 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorymanager.h @@ -26,9 +26,7 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/util/sync.h> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { class MemoryManager; class AllocationLogic; @@ -155,7 +153,4 @@ public: }; -} // defaultimplementation -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp index 125e01db179..a19851ae01f 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp @@ -1,13 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memorystate.h" +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".memory.state"); -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { MemoryState::Entry::Entry() : _currentUsedSize(0), @@ -223,6 +222,4 @@ MemoryState::print(std::ostream& out, bool verbose, out << "\n" << indent << "}"; } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h index 4cbc4da1ac5..5caf5a1d15b 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.h @@ -9,12 +9,10 @@ #pragma once #include "memorymanager.h" -#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/generic/clock/clock.h> #include <vespa/vespalib/util/sync.h> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { class MemoryState : public vespalib::Printable { public: @@ -138,7 +136,4 @@ public: } }; -} // defaultimplementation -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp index 55723eec558..5d5d0b97541 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simplememorylogic.h" - #include <vespa/vespalib/util/exceptions.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".memory.logic.simple"); -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory) : _cacheThreshold(0.98), @@ -232,6 +229,4 @@ SimpleMemoryLogic::print(std::ostream& out, bool verbose, _state.print(out, verbose, indent + " "); } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index 707f6a640a5..eaa66fedc88 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -2,13 +2,12 @@ #include "threadimpl.h" #include "threadpoolimpl.h" +#include <vespa/storageframework/generic/clock/clock.h> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".framework.thread.impl"); -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { ThreadImpl::ThreadImpl(ThreadPoolImpl& pool, Runnable& runnable, @@ -143,6 +142,4 @@ ThreadImpl::AtomicThreadTickData::storeRelaxed( _maxWaitTimeSeenMs.store(newState._maxWaitTimeSeenMs, relaxed); } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index b31b4be3ce9..ad313f510ce 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h @@ -2,14 +2,12 @@ #pragma once +#include <vespa/storageframework/generic/thread/threadpool.h> #include <vespa/vespalib/util/document_runnable.h> -#include <vespa/storageframework/storageframework.h> #include <array> #include <atomic> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { class ThreadPoolImpl; @@ -81,6 +79,4 @@ public: const ThreadProperties& getProperties() const { return _properties; } }; -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index 7f88977bbc5..173ec54f097 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -6,9 +6,7 @@ #include <vespa/fastos/thread.h> #include <vespa/vespalib/util/sync.h> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { class ThreadImpl; @@ -34,7 +32,4 @@ public: Clock& getClock() { return _clock; } }; -} // defaultimplementation -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/clock/clock.h b/storageframework/src/vespa/storageframework/generic/clock/clock.h index c7fd02385ed..69e3cfdc2d0 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/clock.h +++ b/storageframework/src/vespa/storageframework/generic/clock/clock.h @@ -16,8 +16,7 @@ #include "time.h" #include <memory> -namespace storage { -namespace framework { +namespace storage::framework { struct Clock { using UP = std::unique_ptr<Clock>; @@ -32,6 +31,4 @@ struct Clock { virtual MonotonicTimePoint getMonotonicTime() const = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/clock/timer.h b/storageframework/src/vespa/storageframework/generic/clock/timer.h index ea4568fc817..505d41ea23a 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/timer.h +++ b/storageframework/src/vespa/storageframework/generic/clock/timer.h @@ -8,10 +8,9 @@ #pragma once -#include <vespa/storageframework/generic/clock/clock.h> +#include "clock.h" -namespace storage { -namespace framework { +namespace storage::framework { class MilliSecTimer { const Clock* _clock; @@ -37,6 +36,4 @@ public: } }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/component/component.h b/storageframework/src/vespa/storageframework/generic/component/component.h index 5ce86292f7d..95096e23653 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.h +++ b/storageframework/src/vespa/storageframework/generic/component/component.h @@ -74,8 +74,7 @@ #include <vespa/vespalib/util/sync.h> #include <atomic> -namespace storage { -namespace framework { +namespace storage::framework { class ComponentRegister; @@ -217,6 +216,4 @@ public: }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h b/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h index 6934a9fab32..3abec2f3eb2 100644 --- a/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h +++ b/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h @@ -19,8 +19,7 @@ namespace metrics { class Metric; } -namespace storage { -namespace framework { +namespace storage::framework { class StatusReporter; class MemoryManagerInterface; @@ -75,6 +74,4 @@ struct ManagedComponent { }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h b/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h index 94679212179..8bcd5f68c7c 100644 --- a/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h +++ b/storageframework/src/vespa/storageframework/generic/memory/memoryallocationtype.h @@ -20,8 +20,7 @@ #include <string> #include <memory> -namespace storage { -namespace framework { +namespace storage::framework { struct MemoryAllocationType { using UP = std::unique_ptr<MemoryAllocationType>; @@ -49,6 +48,4 @@ private: std::string _name; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h b/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h index b3f3c8ede21..3a9f15b5710 100644 --- a/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h +++ b/storageframework/src/vespa/storageframework/generic/memory/memorymanagerinterface.h @@ -16,8 +16,7 @@ #include "reducememoryusageinterface.h" #include <vector> -namespace storage { -namespace framework { +namespace storage::framework { struct MemoryManagerInterface { @@ -61,6 +60,4 @@ struct MemoryManagerInterface virtual uint64_t getMemorySizeFreeForPriority(uint8_t priority) const = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h index 102656aa4f5..2b132db4434 100644 --- a/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h +++ b/storageframework/src/vespa/storageframework/generic/memory/memorytoken.h @@ -14,8 +14,7 @@ #include <memory> -namespace storage { -namespace framework { +namespace storage::framework { class MemoryToken { protected: @@ -27,6 +26,4 @@ public: virtual bool resize(uint64_t min, uint64_t max) = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h b/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h index 5baaeb2c700..d16d0983bac 100644 --- a/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h +++ b/storageframework/src/vespa/storageframework/generic/memory/reducememoryusageinterface.h @@ -14,8 +14,7 @@ #pragma once -namespace storage { -namespace framework { +namespace storage::framework { struct ReduceMemoryUsageInterface { @@ -36,10 +35,7 @@ struct ReduceMemoryUsageInterface * @param reduceBy Always in the range 0 < reduceBy <= token.size() * @return The amount of memory no longer used. */ - virtual uint64_t reduceMemoryConsumption(const MemoryToken&, - uint64_t reduceBy) = 0; + virtual uint64_t reduceMemoryConsumption(const MemoryToken&, uint64_t reduceBy) = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h b/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h index 07fdf1b5a6e..4efe00bfd9c 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h +++ b/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h @@ -11,8 +11,7 @@ namespace vespalib { class MonitorGuard; } -namespace storage { -namespace framework { +namespace storage::framework { struct MetricUpdateHook { using MetricLockGuard = vespalib::MonitorGuard; @@ -21,6 +20,4 @@ struct MetricUpdateHook { virtual void updateMetrics(const MetricLockGuard &) = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h b/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h index fb9248a97b9..6eedc054a4a 100644 --- a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h +++ b/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h @@ -18,8 +18,7 @@ #include "statusreporter.h" -namespace storage { -namespace framework { +namespace storage::framework { struct HtmlStatusReporter : public StatusReporter { HtmlStatusReporter(vespalib::stringref id, vespalib::stringref name); @@ -62,6 +61,4 @@ struct PartlyHtmlStatusReporter : public HtmlStatusReporter { void reportHtmlStatus(std::ostream&, const HttpUrlPath&) const override {} }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp index e29c66a8865..4c437eaa92e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "httpurlpath.h" +#include <vespa/vespalib/text/stringtokenizer.h> -namespace storage { -namespace framework { +namespace storage::framework { HttpUrlPath::HttpUrlPath(const vespalib::string& urlpath) : _urlPath(urlpath), @@ -69,5 +69,4 @@ HttpUrlPath::print(std::ostream& out, bool, const std::string&) const out << _urlPath; } -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h index 8c8c24c29dd..2704f90d5f1 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h @@ -8,11 +8,10 @@ #include <vespa/vespalib/util/printable.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/text/stringtokenizer.h> #include <map> +#include <sstream> -namespace storage { -namespace framework { +namespace storage::framework { class HttpUrlPath : public vespalib::Printable { vespalib::string _urlPath; @@ -56,6 +55,4 @@ T HttpUrlPath::get(const vespalib::string& id, const T& defaultValue) const return val; } -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/status/statusreporter.h b/storageframework/src/vespa/storageframework/generic/status/statusreporter.h index 0d9aa3a8e4a..8c9cd8c2a2c 100644 --- a/storageframework/src/vespa/storageframework/generic/status/statusreporter.h +++ b/storageframework/src/vespa/storageframework/generic/status/statusreporter.h @@ -18,8 +18,7 @@ #include <vespa/storageframework/generic/status/httpurlpath.h> #include <vespa/vespalib/stllike/string.h> -namespace storage { -namespace framework { +namespace storage::framework { struct StatusReporter { @@ -64,6 +63,4 @@ private: }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h b/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h index 5d7c3383c31..8a8e9c47ae1 100644 --- a/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h +++ b/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h @@ -7,10 +7,11 @@ */ #pragma once -#include <vespa/storageframework/generic/status/statusreporter.h> +#include <vector> -namespace storage { -namespace framework { +namespace storage::framework { + +class StatusReporter; struct StatusReporterMap { virtual ~StatusReporterMap() {} @@ -20,6 +21,4 @@ struct StatusReporterMap { virtual std::vector<const StatusReporter*> getStatusReporters() = 0; }; -} // framework -} // storage - +} // storage::framework diff --git a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h index 98002bf1113..2f38b89fa15 100644 --- a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h +++ b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h @@ -18,10 +18,9 @@ #pragma once #include "statusreporter.h" -#include <vespa/vespalib/util/xmlserializable.h> +#include <vespa/vespalib/util/xmlstream.h> -namespace storage { -namespace framework { +namespace storage::framework { struct XmlStatusReporter : public StatusReporter { XmlStatusReporter(vespalib::stringref id, vespalib::stringref name); @@ -77,6 +76,4 @@ public: } }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/thread/runnable.h b/storageframework/src/vespa/storageframework/generic/thread/runnable.h index 0e6186b546a..f7b295d785b 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/runnable.h +++ b/storageframework/src/vespa/storageframework/generic/thread/runnable.h @@ -11,8 +11,7 @@ #include <vespa/storageframework/generic/clock/time.h> -namespace storage { -namespace framework { +namespace storage::framework { /** * A cycle type can be given when registering ticks. This is useful for @@ -63,6 +62,4 @@ struct Runnable { virtual void run(ThreadHandle&) = 0; }; -} // framework -} // storage - +} // storage::framework diff --git a/storageframework/src/vespa/storageframework/generic/thread/thread.h b/storageframework/src/vespa/storageframework/generic/thread/thread.h index a40a209e980..ed0f34b9421 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/thread.h +++ b/storageframework/src/vespa/storageframework/generic/thread/thread.h @@ -19,8 +19,7 @@ namespace vespalib { class Monitor; } -namespace storage { -namespace framework { +namespace storage::framework { class Thread : public ThreadHandle { vespalib::string _id; @@ -61,6 +60,4 @@ public: void interruptAndJoin(vespalib::Monitor* m); }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h b/storageframework/src/vespa/storageframework/generic/thread/threadpool.h index 1cecd24bdc3..711b782e600 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h +++ b/storageframework/src/vespa/storageframework/generic/thread/threadpool.h @@ -18,8 +18,7 @@ #include <vespa/storageframework/generic/clock/time.h> #include <vector> -namespace storage { -namespace framework { +namespace storage::framework { /** * Each thread may have different properties, as to how long they wait between @@ -94,6 +93,4 @@ struct ThreadPool { virtual void visitThreads(ThreadVisitor&) const = 0; }; -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h index 16a67cf709c..cf2389903f3 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h +++ b/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h @@ -23,8 +23,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/sync.h> -namespace storage { -namespace framework { +namespace storage::framework { class ThreadPool; typedef uint32_t ThreadIndex; @@ -102,5 +101,4 @@ struct TickingThreadPool : public ThreadLock { virtual vespalib::string getStatus() = 0; }; -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/storageframework.h b/storageframework/src/vespa/storageframework/storageframework.h index 9ac5de9fafe..d6c39f5c217 100644 --- a/storageframework/src/vespa/storageframework/storageframework.h +++ b/storageframework/src/vespa/storageframework/storageframework.h @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * \file storageframework.h * * This file includes the most common parts used by the framework. */ diff --git a/storageserver/OWNERS b/storageserver/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/storageserver/OWNERS +++ b/storageserver/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/storageserver/src/apps/storaged/.gitignore b/storageserver/src/apps/storaged/.gitignore index 67fbcdcf4db..282d97fe962 100644 --- a/storageserver/src/apps/storaged/.gitignore +++ b/storageserver/src/apps/storaged/.gitignore @@ -1,3 +1,3 @@ /Makefile /storaged -storaged-bin +vespa-storaged-bin diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt index 8a488b69d77..039058caaff 100644 --- a/storageserver/src/apps/storaged/CMakeLists.txt +++ b/storageserver/src/apps/storaged/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(storageserver_storaged_app SOURCES storage.cpp forcelink.cpp - OUTPUT_NAME storaged-bin + OUTPUT_NAME vespa-storaged-bin INSTALL sbin DEPENDS storageserver_storageapp diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.h b/storageserver/src/vespa/storageserver/app/distributorprocess.h index 8911372b05e..476dd054cd0 100644 --- a/storageserver/src/vespa/storageserver/app/distributorprocess.h +++ b/storageserver/src/vespa/storageserver/app/distributorprocess.h @@ -8,6 +8,7 @@ #pragma once #include "process.h" +#include <vespa/storage/storageserver/distributornode.h> namespace storage { diff --git a/storageserver/src/vespa/storageserver/app/process.cpp b/storageserver/src/vespa/storageserver/app/process.cpp index 352254fba14..c873cb15bb1 100644 --- a/storageserver/src/vespa/storageserver/app/process.cpp +++ b/storageserver/src/vespa/storageserver/app/process.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "process.h" +#include <vespa/storage/storageserver/storagenode.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h index 06e76425fdc..50ceeb43c14 100644 --- a/storageserver/src/vespa/storageserver/app/process.h +++ b/storageserver/src/vespa/storageserver/app/process.h @@ -14,23 +14,29 @@ #pragma once -#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/config/config-documenttypes.h> #include <vespa/storage/storageserver/applicationgenerationfetcher.h> -#include <vespa/storage/storageserver/distributornode.h> -#include <vespa/storage/storageserver/servicelayernode.h> -#include <vespa/config/config.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/subscription/configsubscriber.h> + + +namespace document { class DocumentTypeRepo; } namespace storage { +class StorageNode; +class StorageNodeContext; + class Process : public ApplicationGenerationFetcher { protected: + using DocumentTypeRepoSP = std::shared_ptr<document::DocumentTypeRepo>; config::ConfigUri _configUri; - document::DocumentTypeRepo::SP getTypeRepo() { return _repos.back(); } + DocumentTypeRepoSP getTypeRepo() { return _repos.back(); } config::ConfigSubscriber _configSubscriber; private: config::ConfigHandle<document::DocumenttypesConfig>::UP _documentHandler; - std::vector<document::DocumentTypeRepo::SP> _repos; + std::vector<DocumentTypeRepoSP> _repos; public: typedef std::unique_ptr<Process> UP; diff --git a/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp index c8b40be2404..fa37ed10190 100644 --- a/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpcservicelayerprocess.h" +#include <vespa/storage/config/config-stor-server.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/log/log.h> diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp index 696e79503d7..30b71560287 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "servicelayerprocess.h" +#include <vespa/storage/storageserver/servicelayernode.h> #include <vespa/searchvisitor/searchvisitor.h> namespace storage { @@ -28,4 +29,19 @@ ServiceLayerProcess::createNode() _node->init(); } +StorageNode& +ServiceLayerProcess::getNode() { + return *_node; +} + +StorageNodeContext& +ServiceLayerProcess::getContext() { + return _context; +} + +std::string +ServiceLayerProcess::getComponentName() const { + return "servicelayer"; +} + } // storage diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h index 4140e072a50..dbef3930c20 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h @@ -17,15 +17,21 @@ #pragma once #include "process.h" +#include <vespa/storage/storageserver/servicelayernodecontext.h> +#include <vespa/storage/common/visitorfactory.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> #include <vespa/config-persistence.h> namespace storage { +namespace spi { class PersistenceProvider; } + +class ServiceLayerNode; + class ServiceLayerProcess : public Process { VisitorFactory::Map _externalVisitors; - ServiceLayerNode::UP _node; + std::unique_ptr<ServiceLayerNode> _node; protected: ServiceLayerNodeContext _context; @@ -40,9 +46,9 @@ public: virtual spi::PersistenceProvider& getProvider() = 0; void createNode() override; - StorageNode& getNode() override { return *_node; } - StorageNodeContext& getContext() override { return _context; } - std::string getComponentName() const override { return "servicelayer"; } + StorageNode& getNode() override; + StorageNodeContext& getContext() override; + std::string getComponentName() const override; }; } // storage diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp index dc10ea168c0..4e1ac516a7b 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/feature_resolver.h> #include <vespa/searchvisitor/hitcollector.h> diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index 5ca14fd617d..074070d2ccd 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -2,6 +2,7 @@ #include "hitcollector.h" #include <vespa/searchlib/fef/feature_resolver.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.hitcollector"); diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index d25154aba7f..4befe163377 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -3,7 +3,7 @@ #include "rankmanager.h" #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/functiontablefactory.h> -#include <vespa/vespalib/util/vstringfmt.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.rankmanager"); @@ -106,7 +106,7 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory) LOG(debug, "Number of index environments and rank setups: %u", (uint32_t)_indexEnv.size()); LOG_ASSERT(_properties.size() == _rankSetup.size()); for (uint32_t i = 0; i < _properties.size(); ++i) { - vespalib::string number = vespalib::make_vespa_string("%u", i); + vespalib::string number = vespalib::make_string("%u", i); _rpmap[number] = i; } for (uint32_t i = 0; i < _properties.size(); ++i) { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 616da034c54..7b5e5f17671 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -4,6 +4,7 @@ #include "searchenvironment.h" #include "searchvisitor.h" #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/searchlib/aggregation/modifiers.h> #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/uca/ucaconverter.h> @@ -367,7 +368,7 @@ SearchVisitorFactory::makeVisitor(StorageComponent& component, } void -SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const IteratorContent & c) +SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c) { const document::FieldValue & value = c.getValue(); LOG(debug, "AttributeInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'", @@ -401,13 +402,13 @@ SearchVisitor::PositionInserter::PositionInserter(search::AttributeVector & attr SearchVisitor::PositionInserter::~PositionInserter() {} void -SearchVisitor::PositionInserter::onPrimitive(uint32_t, const IteratorContent & c) +SearchVisitor::PositionInserter::onPrimitive(uint32_t, const Content & c) { (void) c; } void -SearchVisitor::PositionInserter::onStructStart(const IteratorContent & c) +SearchVisitor::PositionInserter::onStructStart(const Content & c) { const document::StructuredFieldValue & value = static_cast<const document::StructuredFieldValue &>(c.getValue()); LOG(debug, "PositionInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'", diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 7767a601ea4..47d6157525c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -27,6 +27,7 @@ #include <vespa/storage/visiting/visitor.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/documentapi/messagebus/messages/queryresultmessage.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> using namespace search::aggregation; @@ -86,13 +87,12 @@ private: * This class gets callbacks when iterating through a field value and * inserts the values into a given attribute vector. **/ - class AttributeInserter : public document::FieldValue::IteratorHandler { + class AttributeInserter : public document::fieldvalue::IteratorHandler { protected: - typedef document::FieldValue::IteratorHandler::Content IteratorContent; search::AttributeVector & _attribute; search::AttributeVector::DocId _docId; - void onPrimitive(uint32_t fid, const IteratorContent & c) override; + void onPrimitive(uint32_t fid, const Content & c) override; public: AttributeInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); @@ -103,7 +103,7 @@ private: PositionInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId); ~PositionInserter(); private: - void onPrimitive(uint32_t fid, const IteratorContent & c) override; + void onPrimitive(uint32_t fid, const Content & c) override; void onStructStart(const Content & fv) override; document::Field _fieldX; document::Field _fieldY; @@ -466,4 +466,3 @@ public: }; } - diff --git a/vdslib/OWNERS b/vdslib/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/vdslib/OWNERS +++ b/vdslib/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/vdslib/src/main/java/com/yahoo/vdslib/loadtype/.gitignore b/vdslib/src/main/java/com/yahoo/vdslib/loadtype/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/loadtype/.gitignore +++ /dev/null diff --git a/vdslib/src/tests/container/documentlisttest.cpp b/vdslib/src/tests/container/documentlisttest.cpp index 5120c4efa05..02e2d0fd935 100644 --- a/vdslib/src/tests/container/documentlisttest.cpp +++ b/vdslib/src/tests/container/documentlisttest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/random.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/testkit/testapp.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/vdslib/src/vespa/vdslib/container/documentlist.cpp b/vdslib/src/vespa/vdslib/container/documentlist.cpp index 2b105a77408..71bee5cf624 100644 --- a/vdslib/src/vespa/vdslib/container/documentlist.cpp +++ b/vdslib/src/vespa/vdslib/container/documentlist.cpp @@ -5,8 +5,9 @@ #include <vespa/document/util/stringutil.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/log/log.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/log/log.h> LOG_SETUP(".vdslib.container.documentlist"); using document::ByteBuffer; diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp index 1c80e3ba8ee..f265a6c947f 100644 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp +++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp @@ -2,6 +2,7 @@ #include "mutabledocumentlist.h" #include <vespa/document/update/documentupdate.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> using vespalib::nbostream; diff --git a/vdslib/src/vespa/vdslib/container/operationlist.cpp b/vdslib/src/vespa/vdslib/container/operationlist.cpp index 6af338a4c0e..8c4515f15ff 100644 --- a/vdslib/src/vespa/vdslib/container/operationlist.cpp +++ b/vdslib/src/vespa/vdslib/container/operationlist.cpp @@ -3,6 +3,7 @@ #include "operationlist.h" #include "documentlist.h" #include <vespa/document/update/documentupdate.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> namespace vdslib { @@ -22,8 +23,7 @@ int OperationList::getRequiredBufferSize() const { switch(_operations[i].opt) { case OperationList::Operation::REMOVE: { - document::Document doc(*document::DataType::DOCUMENT, - _operations[i].docId); + document::Document doc(*document::DataType::DOCUMENT, _operations[i].docId); doc.serializeHeader(stream); break; } diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index d82f89da29d..c685a9b884f 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/xmlstream.h> using namespace vdslib; @@ -60,7 +61,7 @@ void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document: void Parameters::printXml(document::XmlOutputStream& xos) const { - using namespace document; + using namespace vespalib::xml; xos << XmlTag("parameters"); for (const auto & entry : _parameters) { xos << XmlTag("item") diff --git a/vdslib/src/vespa/vdslib/container/visitorordering.h b/vdslib/src/vespa/vdslib/container/visitorordering.h index 3d31a6ac078..e5a61d6332d 100644 --- a/vdslib/src/vespa/vdslib/container/visitorordering.h +++ b/vdslib/src/vespa/vdslib/container/visitorordering.h @@ -1,8 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <string> +#include <cstdint> namespace vdslib { diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index a75ba591af5..cc099164306 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -13,11 +13,10 @@ #include <vespa/config-stor-distribution.h> #include <list> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".vdslib.distribution"); -namespace storage { -namespace lib { +namespace storage::lib { namespace { std::vector<uint32_t> getDistributionBitMasks() { @@ -711,5 +710,4 @@ Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const return result; } -} // lib -} // storage +} diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp index 8ad3dcb2f80..2fce519298f 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.cpp +++ b/vdslib/src/vespa/vdslib/distribution/group.cpp @@ -6,9 +6,9 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <ostream> +#include <cassert> -namespace storage { -namespace lib { +namespace storage::lib { Group::Group(uint16_t index, vespalib::stringref name) : _name(name), @@ -204,4 +204,3 @@ Group::getDistributionConfigHash() const { } } -} diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp index ca7ab742278..0ab4bc6b21a 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp @@ -3,9 +3,9 @@ #include "idealnodecalculatorimpl.h" #include <vespa/vespalib/util/exceptions.h> #include <ostream> +#include <cassert> -namespace storage { -namespace lib { +namespace storage::lib { IdealNodeList::IdealNodeList() : _idealNodes() @@ -71,5 +71,4 @@ IdealNodeCalculatorImpl::initUpStateMapping() { } } -} // lib -} // storage +} diff --git a/vdstestlib/OWNERS b/vdstestlib/OWNERS index 11a58a546b1..dbcff24b338 100644 --- a/vdstestlib/OWNERS +++ b/vdstestlib/OWNERS @@ -1,2 +1 @@ vekterli -dybis diff --git a/vdstestlib/src/tests/cppunit/cppunittest.cpp b/vdstestlib/src/tests/cppunit/cppunittest.cpp index 07c02f6dcdc..49a30e743d5 100644 --- a/vdstestlib/src/tests/cppunit/cppunittest.cpp +++ b/vdstestlib/src/tests/cppunit/cppunittest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> namespace vespalib { diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp index 64ff1377128..5e1c7c6b1b4 100644 --- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp +++ b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <fstream> #include <atomic> diff --git a/vespa-documentgen-plugin/OWNERS b/vespa-documentgen-plugin/OWNERS index 123437e2758..f1a0dc35056 100644 --- a/vespa-documentgen-plugin/OWNERS +++ b/vespa-documentgen-plugin/OWNERS @@ -1 +1,2 @@ -dybis +kraune + diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index de6404f2ea7..6f9e7b91554 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -59,10 +59,6 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <configuration> <updateReleaseInfo>true</updateReleaseInfo> diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 67627ca9410..fff93ec4525 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -10,6 +10,7 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.UnprocessingSearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.tensor.TensorType; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -841,6 +842,9 @@ public class DocumentGenMojo extends AbstractMojo { if (dt instanceof ReferenceDataType) { return "com.yahoo.document.DocumentId"; } + if (dt instanceof TensorDataType) { + return "com.yahoo.tensor.Tensor"; + } return "byte[]"; } @@ -871,7 +875,11 @@ public class DocumentGenMojo extends AbstractMojo { return String.format("new com.yahoo.document.ReferenceDataType(%s.type, %d)", className(((ReferenceDataType) dt).getTargetType().getName()), dt.getId()); } - return "DataType.RAW"; + if (dt instanceof TensorDataType) { + return String.format("new com.yahoo.document.TensorDataType(com.yahoo.tensor.TensorType.fromSpec(\"%s\"))", + ((TensorDataType)dt).getTensorType().toString()); + } + return "com.yahoo.document.DataType.RAW"; } @Override diff --git a/vespa-http-client/OWNERS b/vespa-http-client/OWNERS index efd3ad88b9a..dbcff24b338 100644 --- a/vespa-http-client/OWNERS +++ b/vespa-http-client/OWNERS @@ -1,2 +1 @@ -dybis vekterli diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java index addb762e10e..4a306b1e5de 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java @@ -143,12 +143,10 @@ public class Result { } /** - * @deprecated use getResultType. - * Returns true if the operation was successful. + * Check if operation was successful. * * @return true if the operation was successful. */ - @Deprecated public boolean isSuccess() { return resultType == ResultType.OPERATION_EXECUTED; } diff --git a/vespa_feed_perf/OWNERS b/vespa_feed_perf/OWNERS index 123437e2758..9dc0c2d970d 100644 --- a/vespa_feed_perf/OWNERS +++ b/vespa_feed_perf/OWNERS @@ -1 +1 @@ -dybis +baldersheim diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index b4ed5c69044..4d21e535a42 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_install_script(src/start-cbinaries.sh vdsgetnodestate bin) vespa_install_script(src/start-cbinaries.sh vdsgetsystemstate bin) vespa_install_script(src/start-cbinaries.sh vdssetnodestate bin) vespa_install_script(src/start-cbinaries.sh vdsstat bin) -vespa_install_script(src/start-cbinaries.sh verify_ranksetup bin) +vespa_install_script(src/start-cbinaries.sh vespa-verify-ranksetup bin) vespa_install_script(src/start-cbinaries.sh vespa-config-status bin) vespa_install_script(src/start-cbinaries.sh vespadoclocator bin) vespa_install_script(src/start-cbinaries.sh vespaget bin) @@ -17,13 +17,13 @@ vespa_install_script(src/start-cbinaries.sh vesparoute bin) vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin) vespa_install_script(src/start-cbinaries.sh vespavisit bin) vespa_install_script(src/start-cbinaries.sh vespavisittarget bin) -vespa_install_script(src/start-cbinaries.sh distributord sbin) -vespa_install_script(src/start-cbinaries.sh fdispatch sbin) +vespa_install_script(src/start-cbinaries.sh vespa-vds-disktool bin) +vespa_install_script(src/start-cbinaries.sh vespa-distributord sbin) +vespa_install_script(src/start-cbinaries.sh vespa-fdispatch sbin) vespa_install_script(src/start-cbinaries.sh filedistributor sbin) -vespa_install_script(src/start-cbinaries.sh proton sbin) -vespa_install_script(src/start-cbinaries.sh storaged sbin) +vespa_install_script(src/start-cbinaries.sh vespa-proton sbin) +vespa_install_script(src/start-cbinaries.sh vespa-storaged sbin) -vespa_install_script(src/vespa-allow-downgrade-from-6-to-5.sh vespa-allow-downgrade-from-6-to-5 bin) vespa_install_script(src/start-tool.sh vespa-start-tool.sh bin) vespa_install_script(src/rhel-prestart.sh vespa-prestart.sh bin) diff --git a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh b/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh deleted file mode 100755 index c01333bdd79..00000000000 --- a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -for f in /home/y/var/db/vespa/search/*/*/documents/*/config/config-* -do - echo $f - touch $f/smartsummary.cfg -done diff --git a/vespaclient-container-plugin/OWNERS b/vespaclient-container-plugin/OWNERS index 123437e2758..e030acdbc5b 100644 --- a/vespaclient-container-plugin/OWNERS +++ b/vespaclient-container-plugin/OWNERS @@ -1 +1 @@ -dybis +freva diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 9adb0a09016..dbcb55cfd06 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -16,7 +16,11 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusSyncSession; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; +import com.yahoo.documentapi.metrics.DocumentOperationStatus; +import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.storage.searcher.ContinuationHit; import com.yahoo.vdslib.VisitorOrdering; import com.yahoo.vespaclient.ClusterDef; @@ -28,8 +32,10 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Sends operations to messagebus via document api. @@ -44,6 +50,7 @@ public class OperationHandlerImpl implements OperationHandler { public static final int VISIT_TIMEOUT_MS = 120000; private final DocumentAccess documentAccess; + private final DocumentApiMetrics metricsHelper; private final ClusterEnumerator clusterEnumerator; private static final class SyncSessionFactory extends ResourceFactory<SyncSession> { @@ -59,14 +66,15 @@ public class OperationHandlerImpl implements OperationHandler { private final ConcurrentResourcePool<SyncSession> syncSessions; - public OperationHandlerImpl(DocumentAccess documentAccess) { - this(documentAccess, () -> new ClusterList("client").getStorageClusters()); + public OperationHandlerImpl(DocumentAccess documentAccess, MetricReceiver metricReceiver) { + this(documentAccess, () -> new ClusterList("client").getStorageClusters(), metricReceiver); } - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator) { + public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, MetricReceiver metricReceiver) { this.documentAccess = documentAccess; this.clusterEnumerator = clusterEnumerator; syncSessions = new ConcurrentResourcePool<>(new SyncSessionFactory(documentAccess)); + metricsHelper = new DocumentApiMetrics(metricReceiver, "documentV1"); } @Override @@ -81,11 +89,11 @@ public class OperationHandlerImpl implements OperationHandler { private static final int HTTP_STATUS_INSUFFICIENT_STORAGE = 507; private static final int HTTP_PRE_CONDIDTION_FAILED = 412; - private static int getHTTPStatusCode(DocumentAccessException documentException) { - if (documentException.getErrorCodes().size() == 1 && documentException.getErrorCodes().contains(DocumentProtocol.ERROR_NO_SPACE)) { + public static int getHTTPStatusCode(Set<Integer> errorCodes) { + if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) { return HTTP_STATUS_INSUFFICIENT_STORAGE; } - if (documentException.hasConditionNotMetError()) { + if (errorCodes.contains(DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED)) { return HTTP_PRE_CONDIDTION_FAILED; } return HTTP_STATUS_BAD_REQUEST; @@ -93,10 +101,10 @@ public class OperationHandlerImpl implements OperationHandler { private static Response createErrorResponse(DocumentAccessException documentException, RestUri restUri) { if (documentException.hasConditionNotMetError()) { - return Response.createErrorResponse(getHTTPStatusCode(documentException), "Condition did not match document.", + return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), "Condition did not match document.", restUri, RestUri.apiErrorCodes.DOCUMENT_CONDITION_NOT_MET); } - return Response.createErrorResponse(getHTTPStatusCode(documentException), documentException.getMessage(), restUri, + return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCPETION); } @@ -179,57 +187,79 @@ public class OperationHandlerImpl implements OperationHandler { @Override public void put(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { SyncSession syncSession = syncSessions.alloc(); + Response response; try { + Instant startTime = Instant.now(); DocumentPut put = new DocumentPut(data.getDocument()); put.setCondition(data.getCondition()); setRoute(syncSession, route); syncSession.put(put); + metricsHelper.reportSuccessful(DocumentOperationType.PUT, startTime); + return; } catch (DocumentAccessException documentException) { - throw new RestApiException(createErrorResponse(documentException, restUri)); + response = createErrorResponse(documentException, restUri); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION)); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } + + metricsHelper.reportFailure(DocumentOperationType.PUT, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); + throw new RestApiException(response); } @Override public void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { SyncSession syncSession = syncSessions.alloc(); - setRoute(syncSession, route); + Response response; try { + Instant startTime = Instant.now(); + setRoute(syncSession, route); syncSession.update(data.getDocumentUpdate()); + metricsHelper.reportSuccessful(DocumentOperationType.UPDATE, startTime); + return; } catch (DocumentAccessException documentException) { - throw new RestApiException(createErrorResponse(documentException, restUri)); + response = createErrorResponse(documentException, restUri); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION)); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } + + metricsHelper.reportFailure(DocumentOperationType.UPDATE, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); + throw new RestApiException(response); } @Override public void delete(RestUri restUri, String condition, Optional<String> route) throws RestApiException { SyncSession syncSession = syncSessions.alloc(); - setRoute(syncSession, route); + Response response; try { + Instant startTime = Instant.now(); DocumentId id = new DocumentId(restUri.generateFullId()); DocumentRemove documentRemove = new DocumentRemove(id); + setRoute(syncSession, route); if (condition != null && ! condition.isEmpty()) { documentRemove.setCondition(new TestAndSetCondition(condition)); } syncSession.remove(documentRemove); + metricsHelper.reportSuccessful(DocumentOperationType.REMOVE, startTime); + return; } catch (DocumentAccessException documentException) { if (documentException.hasConditionNotMetError()) { - throw new RestApiException(Response.createErrorResponse(412, "Condition not met: " + documentException.getMessage(), - restUri, RestUri.apiErrorCodes.DOCUMENT_CONDITION_NOT_MET)); + response = Response.createErrorResponse(412, "Condition not met: " + documentException.getMessage(), + restUri, RestUri.apiErrorCodes.DOCUMENT_CONDITION_NOT_MET); + } else { + response = Response.createErrorResponse(400, documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCPETION); } - throw new RestApiException(Response.createErrorResponse(400, documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCPETION)); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); } finally { syncSessions.free(syncSession); } + + metricsHelper.reportFailure(DocumentOperationType.REMOVE, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); + throw new RestApiException(response); } @Override @@ -267,7 +297,7 @@ public class OperationHandlerImpl implements OperationHandler { } if (! wantedCluster.isPresent()) { if (clusters.size() != 1) { - new RestApiException(Response.createErrorResponse(400, "Several clusters exist: " + + throw new RestApiException(Response.createErrorResponse(400, "Several clusters exist: " + clusterListToString(clusters) + " you must specify one. ", RestUri.apiErrorCodes.SEVERAL_CLUSTERS)); } return clusterDefToRoute(clusters.get(0)); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java index 7b6bc4e87c5..26c66085437 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java @@ -105,13 +105,18 @@ public class RestUri { if (readPos >= rawParts.size()) { throwUsage(originalPath); } - return rawParts.get(readPos++); + String nextToken = rawParts.get(readPos++); + return urlDecodeOrException(nextToken); } String restOfPath() throws RestApiException { String rawId = Joiner.on("/").join(rawParts.listIterator(readPos)); + return urlDecodeOrException(rawId); + } + + String urlDecodeOrException(String url) throws RestApiException { try { - return URLDecoder.decode(rawId, StandardCharsets.UTF_8.name()); + return URLDecoder.decode(url, StandardCharsets.UTF_8.name()); } catch (UnsupportedEncodingException e) { throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,"Problems decoding the URI: " + e.getMessage(), apiErrorCodes.ERROR_ID_DECODING_PATH)); } @@ -132,7 +137,8 @@ public class RestUri { case "number": group = Optional.of(new Group(NUMBER_STREAMING, pathParser.nextTokenOrException())); break; - case "docid": group = Optional.empty(); + case "docid": + group = Optional.empty(); break; case "group": group = Optional.of(new Group(GROUP_STREAMING, pathParser.nextTokenOrException())); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 66e8c6475aa..bce8af4593b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -24,6 +24,7 @@ import com.yahoo.document.restapi.RestUri; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -58,12 +59,12 @@ public class RestApi extends LoggingRequestHandler { private final AtomicInteger threadsAvailableForApi; @Inject - public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig, - LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig) { + public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig, + LoadTypeConfig loadTypeConfig, ThreadpoolConfig threadpoolConfig, MetricReceiver metricReceiver) { super(executor, accessLog); MessageBusParams params = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); params.setDocumentmanagerConfig(documentManagerConfig); - this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(params)); + this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(params), metricReceiver); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); // 40% of the threads can be blocked before we deny requests. if (threadpoolConfig != null) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentApiMetrics.java b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentApiMetrics.java new file mode 100644 index 00000000000..13c007a075e --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentApiMetrics.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.metrics; + +import com.yahoo.metrics.simple.Counter; +import com.yahoo.metrics.simple.Gauge; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.metrics.simple.Point; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + + +/** + * This class reports metrics for feed operations by APIs that use documentapi. + * + * @author freva + */ +public class DocumentApiMetrics { + private final Counter feeds; + private final Gauge feedLatency; + private final Map<DocumentOperationStatus, Map<DocumentOperationType, Point>> points = new HashMap<>(); + + public DocumentApiMetrics(MetricReceiver metricReceiver, String apiName) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("api", apiName); + for (DocumentOperationStatus status : DocumentOperationStatus.values()) { + points.put(status, new HashMap<>()); + dimensions.put("status", status.name()); + for (DocumentOperationType operation : DocumentOperationType.values()) { + dimensions.put("operation", operation.name()); + points.get(status).put(operation, new Point(dimensions)); + } + } + + feeds = metricReceiver.declareCounter("feed.operations"); + feedLatency = metricReceiver.declareGauge("feed.latency"); + } + + public void reportSuccessful(DocumentOperationType documentOperationType, double latencyInSeconds) { + Point point = points.get(DocumentOperationStatus.OK).get(documentOperationType); + + feedLatency.sample(latencyInSeconds, point); + feeds.add(point); + } + + public void reportSuccessful(DocumentOperationType documentOperationType, Instant startTime) { + final double latency = Duration.between(startTime, Instant.now()).toMillis() / 1000.0d; + reportSuccessful(documentOperationType, latency); + } + + public void reportFailure(DocumentOperationType documentOperationType, DocumentOperationStatus documentOperationStatus) { + Point point = points.get(documentOperationStatus).get(documentOperationType); + feeds.add(point); + } +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java new file mode 100644 index 00000000000..1513833eee8 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.metrics; + +import com.yahoo.document.restapi.OperationHandlerImpl; + +import java.util.Set; + +/** + * Enum with possible outcomes of a single document feeding operation: + * <ul> + * <li>OK: Document was successfully added/updated/removed</li> + * <li>REQUEST_ERROR: User-made error, for example invalid document format</li> + * <li>SERVER_ERROR: Server-made error, for example insufficient disk space</li> + * </ul> + * + * @author freva + */ +public enum DocumentOperationStatus { + OK, REQUEST_ERROR, SERVER_ERROR; + + public static DocumentOperationStatus fromHttpStatusCode(int httpStatus) { + switch (httpStatus / 100) { + case 2: return OK; + case 4: return REQUEST_ERROR; + case 5: return SERVER_ERROR; + default: return null; + } + } + + public static DocumentOperationStatus fromMessageBusErrorCodes(Set<Integer> errorCodes) { + return fromHttpStatusCode(OperationHandlerImpl.getHTTPStatusCode(errorCodes)); + } +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationType.java b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationType.java new file mode 100644 index 00000000000..8f699c156b4 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationType.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.metrics; + +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.Message; + +/** + * @author freva + */ +public enum DocumentOperationType { + PUT, REMOVE, UPDATE, ERROR; + + public static DocumentOperationType fromMessage(Message msg) { + if (msg instanceof PutDocumentMessage) { + return PUT; + } else if (msg instanceof RemoveDocumentMessage) { + return REMOVE; + } else if (msg instanceof UpdateDocumentMessage) { + return UPDATE; + } else { + return ERROR; + } + } +} + diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index f91038a0a79..8c1344efca1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -6,6 +6,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentTypeManager; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.log.LogLevel; @@ -301,7 +302,7 @@ class ClientFeederV3 { } private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { - msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies)); + msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies, DocumentOperationType.fromMessage(msg.getMessage()))); if (settings.traceLevel != null) { msg.getMessage().getTrace().setLevel(settings.traceLevel); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index fdfc64d7ba4..0412f630469 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -11,16 +11,19 @@ import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.container.logging.AccessLog; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.HttpResponse.Status; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.net.HostName; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; +import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; @@ -56,20 +59,23 @@ public class FeedHandler extends LoggingRequestHandler { private final String localHostname; private final FeedHandlerV3 feedHandlerV3; + @Inject public FeedHandler( Executor executor, DocumentmanagerConfig documentManagerConfig, SessionCache sessionCache, Metric metric, AccessLog accessLog, - ThreadpoolConfig threadpoolConfig) throws Exception { + ThreadpoolConfig threadpoolConfig, + MetricReceiver metricReceiver) throws Exception { super(executor, accessLog); - feedHandlerV3 = new FeedHandlerV3(executor, documentManagerConfig, sessionCache, metric, accessLog, threadpoolConfig); + DocumentApiMetrics metricsHelper = new DocumentApiMetrics(metricReceiver, "vespa.http.server"); + feedHandlerV3 = new FeedHandlerV3(executor, documentManagerConfig, sessionCache, metric, accessLog, threadpoolConfig, metricsHelper); docTypeManager = createDocumentManager(documentManagerConfig); clients = new HashMap<>(); this.sessionCache = sessionCache; sessionId = new AtomicLong(new Random(System.currentTimeMillis()).nextLong()); - feedReplyHandler = new FeedReplyReader(metric); + feedReplyHandler = new FeedReplyReader(metric, metricsHelper); cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron")); cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES); this.metric = metric; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 72fe87ad9b6..2227d0ab41f 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -10,6 +10,7 @@ import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.container.logging.AccessLog; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.log.LogLevel; @@ -53,11 +54,12 @@ public class FeedHandlerV3 extends LoggingRequestHandler { SessionCache sessionCache, Metric metric, AccessLog accessLog, - ThreadpoolConfig threadpoolConfig) throws Exception { + ThreadpoolConfig threadpoolConfig, + DocumentApiMetrics metricsHelper) throws Exception { super(executor, accessLog); docTypeManager = new DocumentTypeManager(documentManagerConfig); this.sessionCache = sessionCache; - feedReplyHandler = new FeedReplyReader(metric); + feedReplyHandler = new FeedReplyReader(metric, metricsHelper); cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron")); cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES); this.metric = metric; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 26dc97c5c65..84a46484cfe 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -1,9 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; +import java.util.Set; import java.util.logging.Logger; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; +import com.yahoo.documentapi.metrics.DocumentOperationStatus; +import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.Reply; @@ -21,9 +25,11 @@ public class FeedReplyReader implements ReplyHandler { private static final Logger log = Logger.getLogger(FeedReplyReader.class.getName()); private final Metric metric; + private final DocumentApiMetrics metricsHelper; - public FeedReplyReader(Metric metric) { + public FeedReplyReader(Metric metric, DocumentApiMetrics metricsHelper) { this.metric = metric; + this.metricsHelper = metricsHelper; } @Override @@ -33,15 +39,18 @@ public class FeedReplyReader implements ReplyHandler { return; } ReplyContext context = (ReplyContext) o; - metric.set( - MetricNames.LATENCY, - Double.valueOf((System.currentTimeMillis() - context.creationTime) / 1000.0d), - null); + final double latencyInSeconds = (System.currentTimeMillis() - context.creationTime) / 1000.0d; + metric.set(MetricNames.LATENCY, latencyInSeconds, null); + if (reply.hasErrors()) { + Set<Integer> errorCodes = reply.getErrorCodes(); + metricsHelper.reportFailure(DocumentOperationType.fromMessage(reply.getMessage()), + DocumentOperationStatus.fromMessageBusErrorCodes(errorCodes)); metric.add(MetricNames.FAILED, 1, null); enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, reply.getError(0).getCode() == DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED, reply.getTrace()); } else { + metricsHelper.reportSuccessful(DocumentOperationType.fromMessage(reply.getMessage()), latencyInSeconds); metric.add(MetricNames.SUCCEEDED, 1, null); enqueue(context, "Document processed.", ErrorCode.OK, false, reply.getTrace()); } @@ -58,5 +67,4 @@ public class FeedReplyReader implements ReplyHandler { Thread.currentThread().interrupt(); } } - }
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java index ad773519458..ff7afb26410 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java @@ -13,6 +13,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.log.LogLevel; @@ -350,7 +351,7 @@ public class Feeder implements Runnable { } private void setMessageParameters(Tuple2<String, Message> msg) { - msg.second.setContext(new ReplyContext(msg.first, feedReplies)); + msg.second.setContext(new ReplyContext(msg.first, feedReplies, DocumentOperationType.fromMessage(msg.second))); if (settings.traceLevel != null) { msg.second.getTrace().setLevel(settings.traceLevel); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java index 30d8b04131d..39392fdefc4 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; +import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.concurrent.BlockingQueue; @@ -8,15 +9,16 @@ import java.util.concurrent.BlockingQueue; /** * Mapping between document ID and client session. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ReplyContext { public final String docId; + public DocumentOperationType documentOperationType; public final BlockingQueue<OperationStatus> feedReplies; public final long creationTime; - public ReplyContext(String docId, BlockingQueue<OperationStatus> feedReplies) { + public ReplyContext(String docId, BlockingQueue<OperationStatus> feedReplies, DocumentOperationType documentOperationType) { this.docId = docId; this.feedReplies = feedReplies; this.creationTime = System.currentTimeMillis(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index e022fa70e31..dd901650264 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -6,6 +6,7 @@ import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespaclient.ClusterDef; import org.junit.Test; @@ -114,7 +115,7 @@ public class OperationHandlerImplTest { return visitorSession; }); OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - return new OperationHandlerImpl(documentAccess, clusterEnumerator); + return new OperationHandlerImpl(documentAccess, clusterEnumerator, MetricReceiver.nullImplementation); } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java index b2a8200b4e1..5dc0cef9e1a 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java @@ -94,6 +94,16 @@ public class RestUriTest { } @Test + public void testGroupUrlDecode() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group#123/myid", "")); + assertThat(restUri.getDocId(), is("myid")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getGroup().get().name, is('g')); + assertThat(restUri.getGroup().get().value, is("group#123")); + assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group#123:myid")); + } + + @Test public void testGroupN() throws Exception { RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", "")); assertThat(restUri.getGroup().get().name, is('n')); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java index 2910b7ad443..792f0321edd 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java @@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.Metric; import com.yahoo.container.logging.AccessLog; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; import com.yahoo.vespa.http.server.FeedHandler; @@ -38,7 +39,7 @@ public class FeedHandlerTest { private final CountDownLatch countDownLatch = new CountDownLatch(1); public TestFeedHandler() throws Exception { - super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null); + super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null, MetricReceiver.nullImplementation); } /** diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java index 7e5ac4dff3a..9fb7dba7257 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -10,10 +10,12 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.feedhandler.NullFeedMetric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespa.http.client.core.ErrorCode; @@ -37,7 +39,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.yahoo.messagebus.Result; import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; public class FeedTesterV3 { @@ -100,7 +101,7 @@ public class FeedTesterV3 { Executor threadPool = Executors.newCachedThreadPool(); DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( - threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null) { + threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null, new DocumentApiMetrics(MetricReceiver.nullImplementation, "test")) { @Override protected ReferencedResource<SharedSourceSession> retainSource( SessionCache sessionCache, SourceSessionParams sessionParams) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java index bdd48949146..022360db54f 100755 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java @@ -1022,7 +1022,8 @@ public class GetSearcherTestCase { this.hitToReturn = hitToReturn; } - public @Override Result search(Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { Result result = new Result(query); result.hits().add(hitToReturn); return result; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java index 2e88c440b12..e6df6b050fe 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java @@ -14,6 +14,7 @@ import com.yahoo.messagebus.*; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; import com.yahoo.text.Utf8String; import com.yahoo.vespa.http.client.core.Headers; @@ -59,7 +60,7 @@ public class V2ErrorsInResultTestCase { private static class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java index 7f52d91cc61..4ec74608548 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java @@ -20,6 +20,7 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; import com.yahoo.vespa.http.client.core.Headers; @@ -50,7 +51,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -115,7 +115,7 @@ public class V2ExternalFeedTestCase { volatile DataFormat lastFormatSeen; public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java index 4e6ca17abef..eca6698de57 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.messagebus.*; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; @@ -60,7 +61,7 @@ public class V2FailingMessagebusTestCase { private class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java index 1b2d855bdf2..e0d9293200d 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java @@ -14,6 +14,7 @@ import com.yahoo.messagebus.*; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; @@ -57,7 +58,7 @@ public class V2NoXmlReaderTestCase { private static class LessConfiguredHandler extends FeedHandler { public LessConfiguredHandler(Executor executor) throws Exception { - super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null, MetricReceiver.nullImplementation); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java index 73095a3efe9..915a8635863 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.References; @@ -15,6 +16,7 @@ import com.yahoo.messagebus.Result; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespaxmlparser.MockFeedReaderFactory; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -86,7 +88,7 @@ public class V3CongestionTestCase { null /*DocTypeManager*/, "clientID", null/*metric*/, - new FeedReplyReader(null/*metric*/), + new FeedReplyReader(null/*metric*/, new DocumentApiMetrics(MetricReceiver.nullImplementation, "tester")), threadsAvail); } diff --git a/vespaclient-core/OWNERS b/vespaclient-core/OWNERS index 123437e2758..e030acdbc5b 100644 --- a/vespaclient-core/OWNERS +++ b/vespaclient-core/OWNERS @@ -1 +1 @@ -dybis +freva diff --git a/vespaclient-java/OWNERS b/vespaclient-java/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/vespaclient-java/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml new file mode 100644 index 00000000000..fb82220cb16 --- /dev/null +++ b/vespaclient-java/pom.xml @@ -0,0 +1,82 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- TODO: Remove this module on Vespa 7 --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>vespaclient-java</artifactId> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespaclient-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>defaults</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>documentapi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java new file mode 100755 index 00000000000..7b104455f15 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -0,0 +1,196 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.dummyreceiver; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.documentapi.ThroughputLimitQueue; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.log.LogSetup; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static java.lang.System.out; + +public class DummyReceiver implements MessageHandler { + String name = null; + DestinationSession session; + MessageBusDocumentAccess da; + long sleepTime = 0; + long messageCount = 0; + int maxPendingCount = 0; + long silentNum = 0; + boolean instant = false; + ThreadPoolExecutor executor = null; + int threads = 10; + long maxQueueTime = -1; + BlockingQueue<Runnable> queue; + boolean verbose = false; + + DummyReceiver() { + } + + public class Task implements Runnable { + Reply reply; + + public Task(Reply reply) { + this.reply = reply; + } + + public void run() { + if (sleepTime > 0) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + session.reply(reply); + } + } + + public void init() { + MessageBusParams params = new MessageBusParams(new LoadTypeSet()); + params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name))); + params.setDocumentManagerConfigId("client"); + params.getMessageBusParams().setMaxPendingCount(maxPendingCount); + params.getMessageBusParams().setMaxPendingSize(0); + da = new MessageBusDocumentAccess(params); + queue = (maxQueueTime < 0) ? new LinkedBlockingDeque<>() : new ThroughputLimitQueue<>(maxQueueTime); + session = da.getMessageBus().createDestinationSession("default", true, this); + executor = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, queue, new DaemonThreadFactory()); + System.out.println("Registered listener at " + name + "/default with " + maxPendingCount + " max pending and sleep time of " + sleepTime); + } + + public void handleMessage(Message message) { + messageCount++; + if ( silentNum == 0 ) { + System.out.println("Received message " + message + ". Received " + messageCount + " messages so far. In queue size " + queue.size()); + + if (verbose) { + if (message instanceof PutDocumentMessage) { + System.out.println(" Document:\n" + ((PutDocumentMessage) message).getDocumentPut().getDocument().toXML(" ")); + } else if (message instanceof RemoveDocumentMessage) { + System.out.println(" Document id: " + ((RemoveDocumentMessage) message).getDocumentId()); + } else if (message instanceof UpdateDocumentMessage) { + System.out.println(" Update:\n " + ((UpdateDocumentMessage) message).getDocumentUpdate().toString()); + } + } + } else { + if ((messageCount % silentNum) == 0) { + System.out.println("Received " + messageCount + " messages so far. In queue size " + queue.size()); + } + } + + EmptyReply reply = new EmptyReply(); + message.swapState(reply); + + if ( ! instant ) { + try { + executor.execute(new Task(reply)); + } catch (RejectedExecutionException e) { + reply.addError(new Error(ErrorCode.SESSION_BUSY, "Session " + name + "/default is busy")); + session.reply(reply); + } + } else { + session.reply(reply); + } + } + + String getParam(List<String> args, String arg) throws IllegalArgumentException { + try { + return args.remove(0); + } catch (Exception e) { + System.err.println("--" + arg + " requires an argument"); + throw new IllegalArgumentException(arg); + } + } + + public void help() { + out.println("Simple receiver for messagebus messages. Prints the messages received to stdout.\n" + + "\n" + + "The options are:\n" + + " --instant Reply in message thread." + + " --name arg Slobrok name to register\n" + + " --maxqueuetime arg Adjust the in queue size to have a maximum queue wait period of this many ms (default -1 = unlimited)\n" + + " --silent #nummsg Do not dump anything, but progress every #nummsg\n" + + " --sleeptime arg The number of milliseconds to sleep per message, to simulate processing time\n" + + " --threads arg The number of threads to process the incoming data\n" + + " --verbose If set, dump the contents of certain messages to stdout"); + } + + boolean parseArgs(List<String> args) { + try { + while (!args.isEmpty()) { + String arg = args.remove(0); + + if (arg.equals("-h") || arg.equals("--help")) { + help(); + return false; + } else if ("--name".equals(arg)) { + name = getParam(args, arg); + } else if ("--sleeptime".equals(arg)) { + sleepTime = Long.parseLong(getParam(args, arg)); + } else if ("--instant".equals(arg)) { + instant = true; + } else if ("--silent".equals(arg)) { + silentNum = Long.parseLong(getParam(args, arg)); + } else if ("--maxqueuetime".equals(arg)) { + maxQueueTime = Long.parseLong(getParam(args, arg)); + } else if ("--threads".equals(arg)) { + threads = Integer.parseInt(getParam(args, arg)); + } else if ("--verbose".equals(arg)) { + verbose = true; + } else { + help(); + return false; + } + } + + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + + public static void main(String[] args) { + LogSetup.initVespaLogging("dummyreceiver"); + DummyReceiver rcv = new DummyReceiver(); + + List<String> l = new LinkedList<>(); + for (String arg : args) { + l.add(arg); + } + if (!rcv.parseArgs(l)) { + System.exit(1); + } + + rcv.init(); + while (true) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java new file mode 100644 index 00000000000..249ed1fd70c --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -0,0 +1,191 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.MessageBusSessionFactory; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.feedapi.SessionFactory; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.io.BufferedOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static java.lang.System.out; + +/** + * Argument parsing class for the vespa feeder. + */ +public class Arguments { + public FeederConfig getFeederConfig() { + return new FeederConfig(feederConfigBuilder); + } + + public List<String> getFiles() { + return files; + } + + public String getMode() { + return mode; + } + + public boolean isVerbose() { + return verbose; + } + + private FeederConfig.Builder feederConfigBuilder = new FeederConfig.Builder(); + private List<String> files = new ArrayList<String>(); + private String dumpDocumentsFile = null; + private String mode = "standard"; + private boolean validateOnly = false; + private boolean verbose = false; + SessionFactory sessionFactory = null; + MessagePropertyProcessor propertyProcessor = null; + private String priority = null; + + public MessagePropertyProcessor getPropertyProcessor() { + return propertyProcessor; + } + + public void help() { + out.println("This is a tool for feeding xml (deprecated) or json data to a Vespa application.\n" + + "\n" + + "The options are:\n" + + " --abortondataerror arg (true) Whether or not to abort if the xml input has \n" + + " errors (true|false).\n" + + " --abortonsenderror arg (true) Whether or not to abort if an error occured while\n" + + " sending operations to Vespa (true|false).\n" + + " --file arg The name of the input files to read. These can \n" + + " also be passed as arguments without the option \n" + + " prefix. If none is given, this tool parses \n" + + " identifiers from standard in.\n" + + " -h [ --help ] Shows this help page.\n" + + " --maxpending arg The maximum number of operations that are allowed\n" + + " to be pending at any given time. NOTE: This disables dynamic throttling. Use with care.\n" + + " --maxpendingsize arg The maximum size (in bytes) of operations that \n" + + " are allowed to be pending at any given time. \n" + + " --maxfeedrate arg Limits the feed rate to the given number (operations/second). You may still want to increase\n" + + " the max pending size if your feed rate doesn't reach the desired number.\n" + + " --mode arg (=standard) The mode to run vespafeeder in (standard | benchmark).\n" + + " --noretry Turns off retries of recoverable failures.\n" + + " --retrydelay arg (=1) The time (in seconds) to wait between retries of \n" + + " a failed operation.\n" + + " --route arg (=default) The route to send the data to.\n" + + " --timeout arg (=180) The time (in seconds) allowed for sending \n" + + " operations.\n" + + " --trace arg (=0) The trace level of network traffic.\n" + + " --validate Run validation tool on input files instead of \n" + + " feeding them.\n" + + " --dumpDocuments <filename> Specify a file where documents in the put are serialized.\n" + + " --priority arg Specify priority of sent messages (see documentation for priority values)\n" + + " --create-if-non-existent Enable setting of create-if-non-existent to true on all document updates in the given xml feed.\n" + + " -v [ --verbose ] Enable verbose output of progress.\n"); + } + + public class HelpShownException extends Exception { + + } + + public Arguments(String[] argList, SessionFactory factory) throws HelpShownException, FileNotFoundException { + parse(argList); + + if (factory != null) { + sessionFactory = factory; + } else if (validateOnly) { + if (dumpDocumentsFile != null) { + BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dumpDocumentsFile)); + sessionFactory = new DummySessionFactory(null, out); + } else { + sessionFactory = new DummySessionFactory(null, null); + } + } else { + sessionFactory = new MessageBusSessionFactory(propertyProcessor); + } + } + + void parse(String[] argList) throws HelpShownException { + List<String> args = new LinkedList<String>(); + args.addAll(Arrays.asList(argList)); + + while (!args.isEmpty()) { + String arg = args.remove(0); + + if (arg.equals("-h") || arg.equals("--help")) { + help(); + throw new HelpShownException(); + } else if ("--abortondataerror".equals(arg)) { + feederConfigBuilder.abortondocumenterror(getBoolean(getParam(args, arg))); + } else if ("--abortonsenderror".equals(arg)) { + feederConfigBuilder.abortonsenderror(getBoolean(getParam(args, arg))); + } else if ("--file".equals(arg)) { + files.add(getParam(args, arg)); + } else if ("--maxpending".equals(arg)) { + feederConfigBuilder.maxpendingdocs(Integer.parseInt(getParam(args, arg))); + } else if ("--maxpendingsize".equals(arg)) { + feederConfigBuilder.maxpendingbytes(Integer.parseInt(getParam(args, arg))); + } else if ("--mode".equals(arg)) { + mode = getParam(args, arg); + } else if ("--noretry".equals(arg)) { + feederConfigBuilder.retryenabled(false); + } else if ("--retrydelay".equals(arg)) { + feederConfigBuilder.retrydelay(Integer.parseInt(getParam(args, arg))); + } else if ("--route".equals(arg)) { + feederConfigBuilder.route(getParam(args, arg)); + } else if ("--timeout".equals(arg)) { + feederConfigBuilder.timeout(Double.parseDouble(getParam(args, arg))); + } else if ("--trace".equals(arg)) { + feederConfigBuilder.tracelevel(Integer.parseInt(getParam(args, arg))); + } else if ("--validate".equals(arg)) { + validateOnly = true; + } else if ("--dumpDocuments".equals(arg)) { + dumpDocumentsFile = getParam(args, arg); + } else if ("--maxfeedrate".equals(arg)) { + feederConfigBuilder.maxfeedrate(Double.parseDouble(getParam(args, arg))); + } else if ("--create-if-non-existent".equals(arg)) { + feederConfigBuilder.createifnonexistent(true); + } else if ("-v".equals(arg) || "--verbose".equals(arg)) { + verbose = true; + } else if ("--priority".equals(arg)) { + priority = getParam(args, arg); + } else { + files.add(arg); + } + } + + propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder())); + } + + private String getParam(List<String> args, String arg) throws IllegalArgumentException { + try { + return args.remove(0); + } catch (Exception e) { + System.err.println("--" + arg + " requires an argument"); + throw new IllegalArgumentException(arg); + } + } + + private Boolean getBoolean(String arg) { + if (arg.equalsIgnoreCase("yes")) { + return true; + } else if (arg.equalsIgnoreCase("no")) { + return false; + } else { + return Boolean.parseBoolean(arg); + } + } + + public String getPriority() { + return priority; + } + + public SessionFactory getSessionFactory() { + return sessionFactory; + } + + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java new file mode 100644 index 00000000000..cc0d8f8c780 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java @@ -0,0 +1,76 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.MessageTypeMetricSet; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.metrics.Metric; +import com.yahoo.metrics.MetricSet; +import com.yahoo.metrics.MetricVisitor; + +import java.io.PrintStream; + +/** + * Class that takes progress from the feed and prints to a stream. + */ +public class BenchmarkProgressPrinter implements RouteMetricSet.ProgressCallback { + private final long startTime; + private final Timer timer; + private final PrintStream output; + + public BenchmarkProgressPrinter(Timer timer, PrintStream output) { + this.timer = timer; + this.output = output; + this.startTime = timer.milliTime(); + } + + class PrintVisitor extends MetricVisitor { + private final PrintStream out; + + PrintVisitor(PrintStream out) { + this.out = out; + } + + @Override + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set instanceof MessageTypeMetricSet && set.getName().equals("total")) { + Metric m = set.getMetric("latency"); + Metric count = set.getMetric("count"); + Metric err = set.getMetric("errors.total"); + + long okCount = 0, errCount = 0, minLatency = 0, maxLatency = 0, avgLatency = 0; + + if (m != null) { + minLatency = m.getLongValue("min"); + maxLatency = m.getLongValue("max"); + avgLatency = m.getLongValue("average"); + } + if (count != null) { + okCount = count.getLongValue("count"); + } + + if (err != null) { + errCount = err.getLongValue("count"); + } + long timeUsed = timer.milliTime() - startTime; + out.println(timeUsed + ", " + okCount + ", " + errCount + ", " + minLatency + ", " + maxLatency + ", " + avgLatency); + } + return true; + } + } + + @Override + public void onProgress(RouteMetricSet metrics) { + //metrics.visit(new PrintVisitor(output), false); + } + + @Override + public void done(RouteMetricSet metrics) { + try { + output.println("# Time used, num ok, num error, min latency, max latency, average latency"); + metrics.visit(new PrintVisitor(output), false); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java new file mode 100755 index 00000000000..3479221257d --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + +public class FileRequest extends InputStreamRequest { + + FileRequest(File f) throws FileNotFoundException { + super(new FileInputStream(f)); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java new file mode 100644 index 00000000000..e69eb6727b0 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.container.jdisc.HttpRequest; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * This is needed because whoever wrote this library moronically decided to pass in-process communication through + * the HTTP layer. As the feeded is being phased out in favor of the standalone HTTP client we don't bother to clean + * it up properly. + * + * @author bratseth + */ +public class InputStreamRequest { + + private InputStream input; + private Map<String, String> properties = new HashMap<>(); + + protected InputStreamRequest(InputStream input) { + this.input = input; + } + + public void setProperty(String key, String value) { + properties.put(key, value); + } + + public String getProperty(String key) { + return properties.get(key); + } + + public HttpRequest toRequest() { + return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.POST, input, properties); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java new file mode 100644 index 00000000000..52087d33a47 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java @@ -0,0 +1,149 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.MessageTypeMetricSet; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.metrics.Metric; +import com.yahoo.metrics.MetricSet; +import com.yahoo.metrics.MetricVisitor; +import com.yahoo.metrics.SumMetric; + +import java.io.IOException; +import java.io.PrintStream; +import java.math.RoundingMode; +import java.text.NumberFormat; +import java.util.Locale; + +/** + * Class that takes progress from the feed and prints to a stream. + */ +public class ProgressPrinter implements RouteMetricSet.ProgressCallback { + private long startTime = 0; + private long lastProgressTime = 0; + private long lastVerboseProgress = 0; + final Timer timer; + final PrintStream output; + + public ProgressPrinter(Timer timer, PrintStream output) { + this.timer = timer; + this.output = output; + + startTime = timer.milliTime(); + lastProgressTime = startTime; + lastVerboseProgress = startTime; + } + + class PrintVisitor extends MetricVisitor { + final PrintStream out; + final NumberFormat format; + + PrintVisitor(PrintStream out) { + this.out = out; + format = NumberFormat.getNumberInstance(Locale.US); + format.setMaximumFractionDigits(2); + format.setMinimumFractionDigits(2); + format.setMinimumIntegerDigits(1); + format.setParseIntegerOnly(false); + format.setRoundingMode(RoundingMode.HALF_UP); + format.setGroupingUsed(false); + } + + @Override + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set instanceof MessageTypeMetricSet && !set.getName().equals("total")) { + Metric m = set.getMetric("latency"); + Metric count = set.getMetric("count"); + Metric err = set.getMetric("errors.total"); + + long okCount = 0, errCount = 0, ignored = 0; + long minLatency = 0, maxLatency = 0, avgLatency = 0; + + if (m != null) { + minLatency = m.getLongValue("min"); + maxLatency = m.getLongValue("max"); + avgLatency = m.getLongValue("average"); + } + if (count != null) { + okCount = count.getLongValue("count"); + } + Metric ignoredMetric = set.getMetric("ignored"); + if (ignoredMetric != null) { + ignored = ignoredMetric.getLongValue("count"); + } + + if (err != null) { + errCount = err.getLongValue("count"); + } + + long timeSinceStart = timer.milliTime() - startTime; + + out.println(((MessageTypeMetricSet)set).getMessageName() + ":\t" + + "ok: " + okCount + + " msgs/sec: " + format.format((double)okCount * 1000 / timeSinceStart) + + " failed: " + errCount + + " ignored: " + ignored + + " latency(min, max, avg): " + minLatency + ", " + maxLatency + ", " + avgLatency); + } + return true; + } + } + + public static String getDashes(int count) { + String dashes = ""; + for (int i = 0; i < count; i++) { + dashes += "-"; + } + + return dashes; + } + + public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) throws IOException { + String headline = "Messages sent to vespa (route " + metrics.getName() + ") :"; + stream.println(headline); + stream.println(getDashes(headline.length())); + metrics.visit(new PrintVisitor(stream), false); + } + + public long getOkMessageCount(RouteMetricSet metrics) { + SumMetric sum = (SumMetric)metrics.getMetric("total"); + + MetricSet ms = (MetricSet)sum.generateSum(); + if (ms != null) { + Metric latency = ms.getMetric("latency"); + if (latency != null) { + return latency.getLongValue("count"); + } + } + + return 0; + } + + @Override + public void onProgress(RouteMetricSet metrics) { + try { + long timeNow = timer.milliTime(); + + if (timeNow - lastVerboseProgress > 30000) { + output.println("\n"); + renderStatusText(metrics, output); + lastVerboseProgress = timeNow; + } else if (timeNow - lastProgressTime > 1000) { + output.print("\rSuccessfully sent " + getOkMessageCount(metrics) + " messages so far"); + lastProgressTime = timeNow; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void done(RouteMetricSet metrics) { + try { + output.println("\n"); + renderStatusText(metrics, output); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java new file mode 100755 index 00000000000..a6ede66c43d --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java @@ -0,0 +1,171 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedhandler.FeedResponse; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.feedhandler.VespaFeedHandler; +import com.yahoo.log.LogSetup; +import com.yahoo.concurrent.SystemTimer; +import com.yahoo.vespaclient.ClusterList; + +import java.io.*; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +public class VespaFeeder { + + Arguments args; + DocumentTypeManager manager; + Executor threadPool = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("vespafeeder")); + + public VespaFeeder(Arguments args, DocumentTypeManager manager) { + this.args = args; + this.manager = manager; + } + + public static class FeedErrorException extends Exception { + String message; + + public FeedErrorException(String message) { + this.message = message; + } + + @Override + public String getMessage() { + return message; + } + + } + + static FeedErrorException renderErrors(List<String> errors) { + StringBuilder buffer = new StringBuilder(); + + if (!errors.isEmpty()) { + String headline = (errors.size() > 10) ? "First 10 errors (of " + errors.size() + "):" : "Errors:"; + buffer.append(headline).append("\n"); + for (int i = 0; i < headline.length(); ++i) { + buffer.append("-"); + } + buffer.append("\n"); + for (int i = 0; i < errors.size() && i < 10; ++i) { + buffer.append(" ").append(errors.get(i)).append("\n"); + } + } + + return new FeedErrorException(buffer.toString()); + } + + public RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) { + if ("benchmark".equals(args.getMode())) { + return new BenchmarkProgressPrinter(SystemTimer.INSTANCE, output); + } else { + return new ProgressPrinter(SystemTimer.INSTANCE, output); + } + } + + void parseFiles(InputStream stdin, PrintStream output) throws Exception { + FeedContext context = new FeedContext( + args.getPropertyProcessor(), + args.getSessionFactory(), + manager, + new ClusterList(), new NullFeedMetric()); + + final BufferedInputStream input = new BufferedInputStream(stdin); + VespaFeedHandler handler = VespaFeedHandler.createFromContext(context, threadPool); + + if (args.getFiles().isEmpty()) { + InputStreamRequest req = new InputStreamRequest(input); + setProperties(req, input); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + if ( ! response.isSuccess()) { + throw renderErrors(response.getErrorList()); + } + } else { + if (args.isVerbose()) { + for (String fileName : args.getFiles()) { + long thisSize = new File(fileName).length(); + output.println("Size of file '" + fileName + "' is " + thisSize + " B."); + } + } + + for (String fileName : args.getFiles()) { + File f = new File(fileName); + FileRequest req = new FileRequest(f); + final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName)); + setProperties(req, inputSnooper); + inputSnooper.close(); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + if (!response.isSuccess()) { + throw renderErrors(response.getErrorList()); + } + } + } + } + + // use BufferedInputStream to enforce the input.markSupported() == true + private void setProperties(InputStreamRequest req, BufferedInputStream input) throws IOException { + setPriority(req); + setCreateIfNonExistent(req); + setJsonInput(req, input); + } + + private void setPriority(InputStreamRequest req) { + if (args.getPriority() != null) { + req.setProperty("priority", args.getPriority()); + } + } + + private void setCreateIfNonExistent(InputStreamRequest req) { + if (args.getFeederConfig().createifnonexistent()) { + req.setProperty("createifnonexistent", "true"); + } + } + + // package access for easy testing + static void setJsonInput(InputStreamRequest req, BufferedInputStream input) throws IOException { + input.mark(4); + int b = input.read(); + input.reset(); + // A valid JSON feed will always start with '[' + if (b == '[') { + req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.TRUE.toString()); + } else { + req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.FALSE.toString()); + } + } + + public static void main(String[] args) { + LogSetup.initVespaLogging("vespafeeder"); + + try { + Arguments arguments = new Arguments(args, null); + + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "client").close(); + + VespaFeeder feeder = new VespaFeeder(arguments, manager); + feeder.parseFiles(System.in, System.out); + System.exit(0); + } catch (Arguments.HelpShownException e) { + System.exit(0); + } catch (IllegalArgumentException e) { + System.exit(1); + } catch (FileNotFoundException e) { + System.err.println("Could not open file " + e.getMessage()); + System.exit(1); + } catch (FeedErrorException e) { + System.err.println("\n" + e.getMessage()); + System.exit(1); + } catch (Exception e) { + System.err.println("Got exception " + e.getMessage() + ", aborting feed."); + System.exit(1); + } + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java new file mode 100644 index 00000000000..b57a9f7bf85 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java @@ -0,0 +1,160 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +import java.util.Iterator; +/** + * This class contains the the program parameters. + * + * @author bjorncs + */ +public class ClientParameters { + + // Determines if the help page should be presented + public final boolean help; + // Contains the document ids. Is backed by either a list iterator if the ids were given as CLI arguments or Scanner(System.in) if ids are provided by standard input. + public final Iterator<String> documentIds; + // Print only the id for retrieved documents + public final boolean printIdsOnly; + // Determines which document fields to retrieve. Default is '[all]'. + public final String fieldSet; + // The Vespa route + public final String route; + // Alternative way to specify the route using cluster name. + public final String cluster; + // The configuration id for message bus. Default "client". + public final String configId; + // Determines if the serialized document size should be printed + public final boolean showDocSize; + // Document request timeout + public final double timeout; + // Determines whether or not the document request can be resent + public final boolean noRetry; + // Vespa trace level + public final int traceLevel; + // Document request priority + public final DocumentProtocol.Priority priority; + // Determines the Vespa load type + public final String loadTypeName; + // If full documents are printed, they will be printed as JSON (instead of XML) + public final boolean jsonOutput; + + + private ClientParameters( + boolean help, Iterator<String> documentIds, boolean printIdsOnly, + String fieldSet, String route, String cluster, String configId, + boolean showDocSize, double timeout, boolean noRetry, int traceLevel, + DocumentProtocol.Priority priority, String loadTypeName, boolean jsonOutput) { + + this.help = help; + this.documentIds = documentIds; + this.printIdsOnly = printIdsOnly; + this.fieldSet = fieldSet; + this.route = route; + this.cluster = cluster; + this.configId = configId; + this.showDocSize = showDocSize; + this.timeout = timeout; + this.noRetry = noRetry; + this.traceLevel = traceLevel; + this.priority = priority; + this.loadTypeName = loadTypeName; + this.jsonOutput = jsonOutput; + } + + public static class Builder { + private boolean help; + private Iterator<String> documentIds; + private boolean printIdsOnly; + private String fieldSet; + private String route; + private String cluster; + private String configId; + private boolean showDocSize; + private double timeout; + private boolean noRetry; + private int traceLevel; + private DocumentProtocol.Priority priority; + private String loadTypeName; + private boolean jsonOutput; + + public Builder setHelp(boolean help) { + this.help = help; + return this; + } + + public Builder setDocumentIds(Iterator<String> documentIds) { + this.documentIds = documentIds; + return this; + } + + public Builder setPrintIdsOnly(boolean printIdsOnly) { + this.printIdsOnly = printIdsOnly; + return this; + } + + public Builder setFieldSet(String fieldSet) { + this.fieldSet = fieldSet; + return this; + } + + public Builder setRoute(String route) { + this.route = route; + return this; + } + + public Builder setCluster(String cluster) { + this.cluster = cluster; + return this; + } + + public Builder setConfigId(String configId) { + this.configId = configId; + return this; + } + + public Builder setShowDocSize(boolean showDocSize) { + this.showDocSize = showDocSize; + return this; + } + + public Builder setTimeout(double timeout) { + this.timeout = timeout; + return this; + } + + public Builder setNoRetry(boolean noRetry) { + this.noRetry = noRetry; + return this; + } + + public Builder setTraceLevel(int traceLevel) { + this.traceLevel = traceLevel; + return this; + } + + public Builder setPriority(DocumentProtocol.Priority priority) { + this.priority = priority; + return this; + } + + public Builder setLoadTypeName(String loadTypeName) { + this.loadTypeName = loadTypeName; + return this; + } + + public Builder setJsonOutput(boolean jsonOutput) { + this.jsonOutput = jsonOutput; + return this; + } + + public ClientParameters build() { + return new ClientParameters( + help, documentIds, printIdsOnly, fieldSet, route, cluster, configId, + showDocSize, timeout, noRetry, traceLevel, priority, loadTypeName, jsonOutput); + } + } + + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java new file mode 100644 index 00000000000..cbaef17a70a --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -0,0 +1,263 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Scanner; + +/** + * This class is responsible for parsing the command line arguments and print the help page. + * + * @author bjorncs + */ +public class CommandLineOptions { + + public static final String HELP_OPTION = "help"; + public static final String PRINTIDS_OPTION = "printids"; + public static final String HEADERSONLY_OPTION = "headersonly"; + public static final String FIELDSET_OPTION = "fieldset"; + public static final String CLUSTER_OPTION = "cluster"; + public static final String ROUTE_OPTION = "route"; + public static final String CONFIGID_OPTION = "configid"; + public static final String SHOWDOCSIZE_OPTION = "showdocsize"; + public static final String TIMEOUT_OPTION = "timeout"; + public static final String NORETRY_OPTION = "noretry"; + public static final String TRACE_OPTION = "trace"; + public static final String PRIORITY_OPTION = "priority"; + public static final String LOADTYPE_OPTION = "loadtype"; + public static final String JSONOUTPUT_OPTION = "jsonoutput"; + + private final Options options = createOptions(); + private final InputStream stdIn; + + public CommandLineOptions(InputStream stdIn) { + this.stdIn = stdIn; + } + + public CommandLineOptions() { + this(System.in); + } + + @SuppressWarnings("AccessStaticViaInstance") + private static Options createOptions() { + Options options = new Options(); + + options.addOption(Option.builder("h") + .hasArg(false) + .desc("Show this syntax page.") + .longOpt(HELP_OPTION) + .build()); + + options.addOption(Option.builder("i") + .hasArg(false) + .desc("Show only identifiers of retrieved documents.") + .longOpt(PRINTIDS_OPTION) + .build()); + + options.addOption(Option.builder("e") + .hasArg(false) + .desc("Retrieve header fields only. [Deprecated].") + .longOpt(HEADERSONLY_OPTION).build()); + + options.addOption(Option.builder("f") + .hasArg(true) + .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html) (default '[all]')") + .longOpt(FIELDSET_OPTION) + .argName("fieldset").build()); + + options.addOption(Option.builder("u") + .hasArg(true) + .desc("Send request to the given content cluster.") + .longOpt(CLUSTER_OPTION) + .argName("cluster").build()); + + options.addOption(Option.builder("r") + .hasArg(true) + .desc("Send request to the given messagebus route.") + .longOpt(ROUTE_OPTION) + .argName("route").build()); + + options.addOption(Option.builder("c") + .hasArg(true) + .desc("Use the specified config id for messagebus configuration.") + .longOpt(CONFIGID_OPTION) + .argName("configid").build()); + + options.addOption(Option.builder("s") + .hasArg(false) + .desc("Show binary size of document.") + .longOpt(SHOWDOCSIZE_OPTION).build()); + + options.addOption(Option.builder("t") + .hasArg(true) + .desc("Set timeout for the request in seconds (default 0).") + .longOpt(TIMEOUT_OPTION) + .argName("timeout") + .type(Number.class).build()); + + options.addOption(Option.builder("n") + .hasArg(false) + .desc("Do not retry operation on transient errors, as is default.") + .longOpt(NORETRY_OPTION).build()); + + options.addOption(Option.builder("a") + .hasArg(true) + .desc("Trace level to use (default 0).") + .longOpt(TRACE_OPTION) + .argName("trace") + .type(Number.class).build()); + + options.addOption(Option.builder("p") + .hasArg(true) + .desc("Priority (default 6).") + .longOpt(PRIORITY_OPTION) + .argName("priority") + .type(Number.class).build()); + + options.addOption(Option.builder("l") + .hasArg(true) + .desc("Load type (default \"\").") + .longOpt(LOADTYPE_OPTION) + .argName("loadtype").build()); + + options.addOption(Option.builder("j") + .hasArg(false) + .desc("JSON output") + .longOpt(JSONOUTPUT_OPTION).build()); + + return options; + } + + public void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + + formatter.printHelp( + "vespaget <options> [documentid...]", "Fetch a document from a Vespa Content cluster.", options, + "If one or more document identifier are specified, these documents will be " + + "retrieved. Otherwise, document identifiers (separated with line break) will be read from standard in.\n", + false); + } + + public ClientParameters parseCommandLineArguments(String[] args) throws IllegalArgumentException { + try { + CommandLineParser clp = new DefaultParser(); + CommandLine cl = clp.parse(options, args); + + boolean printIdsOnly = cl.hasOption(PRINTIDS_OPTION); + boolean headersOnly = cl.hasOption(HEADERSONLY_OPTION); + String fieldSet = cl.getOptionValue(FIELDSET_OPTION, ""); + String cluster = cl.getOptionValue(CLUSTER_OPTION, ""); + String route = cl.getOptionValue(ROUTE_OPTION, ""); + String configId = cl.getOptionValue(CONFIGID_OPTION, ""); + boolean help = cl.hasOption(HELP_OPTION); + String loadtype = cl.getOptionValue(LOADTYPE_OPTION, ""); + boolean noRetry = cl.hasOption(NORETRY_OPTION); + boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION); + boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); + int trace = getTrace(cl); + DocumentProtocol.Priority priority = getPriority(cl); + double timeout = getTimeout(cl); + Iterator<String> documentIds = getDocumentIds(cl); + + if (printIdsOnly && headersOnly) { + throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive."); + } + if ((printIdsOnly || headersOnly) && !fieldSet.isEmpty()) { + throw new IllegalArgumentException("Field set option can not be used in combination with print ids or headers only options."); + } + + if (printIdsOnly) { + fieldSet = "[id]"; + } else if (headersOnly) { + fieldSet = "[header]"; + } else if (fieldSet.isEmpty()) { + fieldSet = "[all]"; + } + + if (!cluster.isEmpty() && !route.isEmpty()) { + throw new IllegalArgumentException("Cluster and route options are mutually exclusive."); + } + + if (route.isEmpty() && cluster.isEmpty()) { + route = "default"; + } + + if (trace < 0 || trace > 9) { + throw new IllegalArgumentException("Invalid tracelevel: " + trace); + } + + if (configId.isEmpty()) { + configId = "client"; + } + + ClientParameters.Builder paramsBuilder = new ClientParameters.Builder(); + return paramsBuilder + .setDocumentIds(documentIds) + .setConfigId(configId) + .setFieldSet(fieldSet) + .setHelp(help) + .setPrintIdsOnly(printIdsOnly) + .setLoadTypeName(loadtype) + .setNoRetry(noRetry) + .setCluster(cluster) + .setRoute(route) + .setShowDocSize(showDocSize) + .setTraceLevel(trace) + .setPriority(priority) + .setTimeout(timeout) + .setJsonOutput(jsonOutput) + .build(); + } catch (ParseException pe) { + throw new IllegalArgumentException(pe.getMessage()); + } + } + + private Iterator<String> getDocumentIds(CommandLine cl) { + // Fetch document ids from stdin if no ids are passed in as command line arguments + List<String> documentIds = Arrays.asList(cl.getArgs()); + // WARNING: CommandLine.getArgs may return a single empty string as the only element + if (documentIds.isEmpty() || + documentIds.size() == 1 && documentIds.get(0).isEmpty()) { + return new Scanner(stdIn); + } else { + return documentIds.iterator(); + } + } + + private static double getTimeout(CommandLine cl) throws ParseException { + Number timeoutObj = (Number) cl.getParsedOptionValue(TIMEOUT_OPTION); + return timeoutObj != null ? timeoutObj.doubleValue() : 0; + } + + private static int getTrace(CommandLine cl) throws ParseException { + Number traceObj = (Number) cl.getParsedOptionValue(TRACE_OPTION); + return traceObj != null ? traceObj.intValue() : 0; + } + + private static DocumentProtocol.Priority getPriority(CommandLine cl) throws ParseException { + Number priorityObj = (Number) cl.getParsedOptionValue(PRIORITY_OPTION); + int priorityNumber = priorityObj != null ? priorityObj.intValue() : DocumentProtocol.Priority.NORMAL_2.getValue(); + return parsePriority(priorityNumber); + } + + private static DocumentProtocol.Priority parsePriority(int n) { + for (DocumentProtocol.Priority priority : DocumentProtocol.Priority.values()) { + if (priority.getValue() == n) { + return priority; + } + } + throw new IllegalArgumentException("Invalid priority: " + n); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java new file mode 100644 index 00000000000..6836f033c11 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; + +/** + * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}. + * + * @author bjorncs + */ +public class DocumentAccessFactory { + + public MessageBusDocumentAccess createDocumentAccess(MessageBusParams messageBusParams) { + return new MessageBusDocumentAccess(messageBusParams); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java new file mode 100644 index 00000000000..6e52e89c580 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -0,0 +1,207 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Trace; +import com.yahoo.text.Utf8; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; + +import java.util.Iterator; +import java.util.Map; + +/** + * The document retriever is responsible for retrieving documents using the Document API and printing the result to standard out. + * + * @author bjorncs + */ +public class DocumentRetriever { + + private final ClusterList clusterList; + private final DocumentAccessFactory documentAccessFactory; + private final ClientParameters params; + private final LoadTypeSet loadTypeSet; + + private MessageBusSyncSession session; + private MessageBusDocumentAccess documentAccess; + + public DocumentRetriever(ClusterList clusterList, + DocumentAccessFactory documentAccessFactory, + LoadTypeSet loadTypeSet, + ClientParameters params) { + this.clusterList = clusterList; + this.documentAccessFactory = documentAccessFactory; + this.loadTypeSet = loadTypeSet; + this.params = params; + } + + public void shutdown() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException e) { + // Ignore exception on shutdown + } + try { + if (documentAccess != null) { + documentAccess.shutdown(); + } + } catch (IllegalStateException e) { + // Ignore exception on shutdown + } + } + + public void retrieveDocuments() throws DocumentRetrieverException { + boolean first = true; + String route = params.cluster.isEmpty() ? params.route : resolveClusterRoute(params.cluster); + LoadType loadType = params.loadTypeName.isEmpty() ? null : resolveLoadType(params.loadTypeName); + + MessageBusParams messageBusParams = createMessageBusParams(params.configId, params.timeout, route); + documentAccess = documentAccessFactory.createDocumentAccess(messageBusParams); + session = documentAccess.createSyncSession(new SyncParameters()); + int trace = params.traceLevel; + if (trace > 0) { + session.setTraceLevel(trace); + } + + Iterator<String> iter = params.documentIds; + if (params.jsonOutput && !params.printIdsOnly) { + System.out.println('['); + } + while (iter.hasNext()) { + if (params.jsonOutput && !params.printIdsOnly) { + if (!first) { + System.out.println(','); + } else { + first = false; + } + } + String docid = iter.next(); + Message msg = createDocumentRequest(docid, loadType); + Reply reply = session.syncSend(msg); + printReply(reply); + } + if (params.jsonOutput && !params.printIdsOnly) { + System.out.println(']'); + } + } + + private String resolveClusterRoute(String clusterName) throws DocumentRetrieverException { + if (clusterList.getStorageClusters().isEmpty()) { + throw new DocumentRetrieverException("The Vespa cluster does not have any content clusters declared."); + } + + ClusterDef clusterDef = null; + for (ClusterDef c : clusterList.getStorageClusters()) { + if (c.getName().equals(clusterName)) { + clusterDef = c; + } + } + if (clusterDef == null) { + String names = createClusterNamesString(); + throw new DocumentRetrieverException(String.format( + "The Vespa cluster contains the content clusters %s, not %s. Please select a valid vespa cluster.", + names, clusterName)); + } + return String.format("[Storage:cluster=%s;clusterconfigid=%s]", clusterDef.getName(), clusterDef.getConfigId()); + } + + private LoadType resolveLoadType(String loadTypeName) throws DocumentRetrieverException { + Map<String, LoadType> loadTypesNameMap = loadTypeSet.getNameMap(); + if (!loadTypesNameMap.containsKey(loadTypeName)) { + throw new DocumentRetrieverException(String.format("Loadtype with name '%s' does not exist.\n", loadTypeName)); + } else { + return loadTypesNameMap.get(loadTypeName); + } + } + + private MessageBusParams createMessageBusParams(String configId, double timeout, String route) { + MessageBusParams messageBusParams = new MessageBusParams(loadTypeSet); + messageBusParams.setRoute(route); + messageBusParams.setProtocolConfigId(configId); + messageBusParams.setRoutingConfigId(configId); + messageBusParams.setDocumentManagerConfigId(configId); + + if (timeout > 0) { + messageBusParams.getSourceSessionParams().setTimeout(timeout); + } + return messageBusParams; + } + + private Message createDocumentRequest(String docid, LoadType loadType) { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId(docid), params.fieldSet); + msg.setPriority(params.priority); + msg.setRetryEnabled(!params.noRetry); + + if (loadType != null) { + msg.setLoadType(loadType); + } + return msg; + } + + private void printReply(Reply reply) { + Trace trace = reply.getTrace(); + if (!trace.getRoot().isEmpty()) { + System.out.println(trace); + } + + if (reply.hasErrors()) { + System.err.print("Request failed: "); + for (int i = 0; i < reply.getNumErrors(); i++) { + System.err.printf("\n %s", reply.getError(i)); + } + System.err.println(); + return; + } + + if (!(reply instanceof GetDocumentReply)) { + System.err.printf("Unexpected reply %s: '%s'\n", reply.getType(), reply.toString()); + return; + } + + GetDocumentReply documentReply = (GetDocumentReply) reply; + Document document = documentReply.getDocument(); + + if (document == null) { + System.out.println("Document not found."); + return; + } + + if (params.showDocSize) { + System.out.printf("Document size: %d bytes.\n", document.getSerializedSize()); + } + if (params.printIdsOnly) { + System.out.println(document.getId()); + } else { + if (params.jsonOutput) { + System.out.print(Utf8.toString(JsonWriter.toByteArray(document))); + } else { + System.out.print(document.toXML(" ")); + } + } + } + + private String createClusterNamesString() { + StringBuilder names = new StringBuilder(); + for (ClusterDef c : clusterList.getStorageClusters()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(c.getName()); + } + return names.toString(); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java new file mode 100644 index 00000000000..4cf0e9885a3 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +/** + * Exception thrown by {@link DocumentRetriever}. + * + * @author bjorncs + */ +public class DocumentRetrieverException extends Exception { + + public DocumentRetrieverException(String message) { + super(message); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java new file mode 100644 index 00000000000..324107d8909 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java @@ -0,0 +1,46 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + + +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespaclient.ClusterList; + +/** + * The vespaget tool retrieves documents from a Vespa Document Storage cluster, and prints them to stdout as XML. + * + * @author bjorncs + */ +public class Main { + + public static void main(String[] args) { + try { + CommandLineOptions options = new CommandLineOptions(); + ClientParameters params = options.parseCommandLineArguments(args); + + if (params.help) { + options.printHelp(); + } else { + DocumentRetriever documentRetriever = createDocumentRetriever(params); + addShutdownHook(documentRetriever); + documentRetriever.retrieveDocuments(); + } + } catch (IllegalArgumentException e) { + System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage()); + } catch (DocumentRetrieverException e) { + System.err.printf("Failed to retrieve documents: %s\n", e.getMessage()); + } + } + + private static void addShutdownHook(DocumentRetriever documentRetriever) { + Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown)); + } + + private static DocumentRetriever createDocumentRetriever(ClientParameters params) { + return new DocumentRetriever( + new ClusterList("client"), + new DocumentAccessFactory(), + new LoadTypeSet(params.configId), + params + ); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java new file mode 100644 index 00000000000..a6f471e7b5e --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * Exception class used by {@link com.yahoo.vespastat.BucketStatsRetriever}. + * + * @author bjorncs + */ +public class BucketStatsException extends Exception { + public BucketStatsException(String message) { + super(message); + } + + public BucketStatsException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java new file mode 100644 index 00000000000..a4a263188b5 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; + +import java.io.PrintStream; +import java.util.List; + +/** + * The class is responsible for printing bucket information to a printstream. + * + * @author bjorncs + */ +public class BucketStatsPrinter { + private final BucketStatsRetriever retriever; + private final PrintStream out; + + public BucketStatsPrinter( + BucketStatsRetriever retriever, + PrintStream out) { + this.retriever = retriever; + this.out = out; + } + + public void retrieveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException { + BucketId bucketId = retriever.getBucketIdForType(type, id); + if (type == ClientParameters.SelectionType.GROUP || type == ClientParameters.SelectionType.USER) { + out.printf("Generated 32-bit bucket id: %s\n", bucketId); + } + + List<GetBucketListReply.BucketInfo> bucketList = retriever.retrieveBucketList(bucketId); + printBucketList(bucketList); + + if (dumpData) { + for (GetBucketListReply.BucketInfo bucketInfo : bucketList) { + BucketId bucket = bucketInfo.getBucketId(); + String bucketStats = retriever.retrieveBucketStats(type, id, bucket); + printBucketStats(bucket, bucketStats); + } + } + } + + private void printBucketList(List<GetBucketListReply.BucketInfo> bucketList) { + if (bucketList.isEmpty()) { + out.println("No actual files were stored for this bucket."); + } else { + out.println("Bucket maps to the following actual files:"); + for (GetBucketListReply.BucketInfo bucketInfo : bucketList) { + out.printf("\t%s\n", bucketInfo); + } + } + } + + private void printBucketStats(BucketId bucket, String stats) { + out.printf("\nDetails for %s:\n%s", bucket, stats); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java new file mode 100644 index 00000000000..84e89349f9f --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java @@ -0,0 +1,176 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.document.select.BucketSelector; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.routing.Route; + +import java.util.List; + +/** + * This class fetches bucket information from Vespa + * + * @author bjorncs + */ +public class BucketStatsRetriever { + + private final BucketIdFactory bucketIdFactory = new BucketIdFactory(); + private final BucketSelector selector = new BucketSelector(bucketIdFactory); + + private final MessageBusSyncSession session; + private final MessageBusDocumentAccess documentAccess; + private final String route; + + public BucketStatsRetriever( + DocumentAccessFactory documentAccessFactory, + String route, + ShutdownHookRegistrar registrar) { + registerShutdownHook(registrar); + this.documentAccess = documentAccessFactory.createDocumentAccess(); + this.session = documentAccess.createSyncSession(new SyncParameters()); + this.route = route; + } + + private void registerShutdownHook(ShutdownHookRegistrar registrar) { + registrar.registerShutdownHook(() -> { + try { + session.destroy(); + } catch (Exception e) { + // Ignore exception on shutdown + } + try { + documentAccess.shutdown(); + } catch (Exception e) { + // Ignore exception on shutdown + } + }); + } + + public BucketId getBucketIdForType(ClientParameters.SelectionType type, String id) throws BucketStatsException { + switch (type) { + case DOCUMENT: + return bucketIdFactory.getBucketId(new DocumentId(id)); + case BUCKET: + // The internal parser of BucketID is used since the Java Long.decode cannot handle unsigned longs. + return new BucketId(String.format("BucketId(%s)", id)); + case GID: + return convertGidToBucketId(id); + case USER: + case GROUP: + try { + BucketSet bucketList = selector.getBucketList(createDocumentSelection(type, id)); + if (bucketList.size() != 1) { + String message = String.format("Document selection must map to only one location. " + + "Specified selection matches %d locations.", bucketList.size()); + throw new BucketStatsException(message); + } + return bucketList.iterator().next(); + } catch (ParseException e) { + throw new BucketStatsException(String.format("Invalid id: %s (%s).", id, e.getMessage()), e); + } + default: + throw new RuntimeException("Unreachable code"); + } + } + + public String retrieveBucketStats(ClientParameters.SelectionType type, String id, BucketId bucketId) throws BucketStatsException { + String documentSelection = createDocumentSelection(type, id); + StatBucketMessage msg = new StatBucketMessage(bucketId, documentSelection); + StatBucketReply statBucketReply = sendMessage(msg, StatBucketReply.class); + return statBucketReply.getResults(); + } + + public List<GetBucketListReply.BucketInfo> retrieveBucketList(BucketId bucketId) throws BucketStatsException { + GetBucketListMessage msg = new GetBucketListMessage(bucketId); + GetBucketListReply bucketListReply = sendMessage(msg, GetBucketListReply.class); + return bucketListReply.getBuckets(); + } + + + private <T extends Reply> T sendMessage(DocumentMessage msg, Class<T> expectedReply) throws BucketStatsException { + setRoute(msg, route); + Reply reply = session.syncSend(msg); + return validateReply(reply, expectedReply); + } + + private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException { + try { + msg.setRoute(Route.parse(route)); + } catch (Exception e) { + throw new BucketStatsException(String.format("Invalid route: '%s'.", route)); + } + } + + private static <T extends Reply> T validateReply(Reply reply, Class<T> type) throws BucketStatsException { + if (reply.hasErrors()) { + throw new BucketStatsException(makeErrorMessage(reply)); + } + if (!type.isInstance(reply)) { + throw new BucketStatsException(String.format("Unexpected reply %s: '%s'", reply.getType(), reply.toString())); + } + return type.cast(reply); + } + + private static String makeErrorMessage(Reply reply) { + StringBuilder b = new StringBuilder(); + b.append("Request failed: \n"); + for (int i = 0; i < reply.getNumErrors(); i++) { + b.append(String.format("\t %s\n", reply.getError(i))); + } + return b.toString(); + } + + private static String createDocumentSelection(ClientParameters.SelectionType type, String id) { + switch (type) { + case BUCKET: + return "true"; + case DOCUMENT: + return String.format("id=\"%s\"", id); + case GID: + return String.format("id.gid=\"gid(%s)\"", id); + case USER: + return String.format("id.user=%s", id); + case GROUP: + return String.format("id.group=\"%s\"", id); + default: + throw new RuntimeException("Unreachable code"); + } + } + + private static BucketId convertGidToBucketId(String id) throws BucketStatsException { + if (!id.matches("0x\\p{XDigit}{24}")) { + throw new BucketStatsException("Invalid gid: " + id); + } + String hexWithoutPrefix = id.substring(2); + return new GlobalId(convertHexStringToByteArray(hexWithoutPrefix)).toBucketId(); + } + + private static byte[] convertHexStringToByteArray(String s) throws BucketStatsException { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + int digit1 = Character.digit(s.charAt(i), 16); + int digit2 = Character.digit(s.charAt(i + 1), 16); + data[i / 2] = (byte) ((digit1 << 4) + digit2); + } + return data; + } + + public interface ShutdownHookRegistrar { + void registerShutdownHook(Runnable runnable); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java new file mode 100644 index 00000000000..bba0d9803ed --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * This class contains the program parameters. + * + * @author bjorncs + */ +public class ClientParameters { + // Show help page if true + public final boolean help; + // Dump list of documents for all buckets matching the selection if true + public final boolean dumpData; + // The message bus route + public final String route; + // The selection type + public final SelectionType selectionType; + // The selection id + public final String id; + + public ClientParameters( + boolean help, + boolean dumpData, + String route, + SelectionType selectionType, + String id) { + this.help = help; + this.dumpData = dumpData; + this.route = route; + this.selectionType = selectionType; + this.id = id; + } + + public enum SelectionType {USER, GROUP, BUCKET, GID, DOCUMENT} + + public static class Builder { + private boolean help; + private boolean dumpData; + private String route; + private SelectionType selectionType; + private String id; + + public Builder setHelp(boolean help) { + this.help = help; + return this; + } + + public Builder setDumpData(boolean dumpData) { + this.dumpData = dumpData; + return this; + } + + public Builder setRoute(String route) { + this.route = route; + return this; + } + + public Builder setSelectionType(SelectionType selectionType) { + this.selectionType = selectionType; + return this; + } + + public Builder setId(String id) { + this.id = id; + return this; + } + + public ClientParameters build() { + return new ClientParameters(help, dumpData, route, selectionType, id); + } + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java new file mode 100644 index 00000000000..b0b6246a262 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java @@ -0,0 +1,139 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * Responsible for parsing the command line arguments and presenting the help page + * + * @author bjorncs + */ +public class CommandLineOptions { + + private static final String HELP_OPTION = "help"; + private static final String DUMP_OPTION = "dump"; + private static final String ROUTE_OPTION = "route"; + private static final String USER_OPTION = "user"; + private static final String GROUP_OPTION = "group"; + private static final String BUCKET_OPTION = "bucket"; + private static final String GID_OPTION = "gid"; + private static final String DOCUMENT_OPTION = "document"; + + private final Options options = createOptions(); + + @SuppressWarnings("AccessStaticViaInstance") + private static Options createOptions() { + Options options = new Options(); + + options.addOption(Option.builder("h") + .hasArg(false) + .desc("Show this syntax page.") + .longOpt(HELP_OPTION) + .build()); + + options.addOption(Option.builder("d") + .hasArg(false) + .desc("Dump list of documents for all buckets matching the selection command.") + .longOpt(DUMP_OPTION) + .build()); + + options.addOption(Option.builder("r") + .hasArg(true) + .desc("Route to send the messages to, usually the name of the storage cluster.") + .argName("route") + .longOpt(ROUTE_OPTION) + .build()); + + // A group of mutually exclusive options for user, group, bucket, gid and document. + OptionGroup optionGroup = new OptionGroup(); + optionGroup.setRequired(false); + + optionGroup.addOption(Option.builder("u") + .hasArg(true) + .desc("Dump list of buckets that can contain the given user.") + .argName("userid") + .longOpt(USER_OPTION) + .build()); + + optionGroup.addOption(Option.builder("g") + .hasArg(true) + .desc("Dump list of buckets that can contain the given group.") + .argName("groupid") + .longOpt(GROUP_OPTION) + .build()); + + optionGroup.addOption(Option.builder("b") + .hasArg(true) + .desc("Dump list of buckets that are contained in the given bucket, or that contain it.") + .argName("bucketid") + .longOpt(BUCKET_OPTION) + .build()); + + optionGroup.addOption(Option.builder("l") + .hasArg(true) + .desc("Dump information about one specific document, as given by the GID (implies --dump).") + .argName("globalid") + .longOpt(GID_OPTION) + .build()); + + optionGroup.addOption(Option.builder("o") + .hasArg(true) + .desc("Dump information about one specific document (implies --dump).") + .argName("docid") + .longOpt(DOCUMENT_OPTION) + .build()); + + options.addOptionGroup(optionGroup); + return options; + } + + public void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vdsstat [options]", + "Fetch statistics about a specific user, group, bucket, gid or document.", options, "", false); + } + + public ClientParameters parseCommandLineArguments(String[] args) { + try { + CommandLineParser clp = new DefaultParser(); + CommandLine cl = clp.parse(options, args); + ClientParameters.Builder builder = new ClientParameters.Builder(); + + builder.setHelp(cl.hasOption(HELP_OPTION)); + builder.setDumpData(cl.hasOption(DUMP_OPTION)); + builder.setRoute(cl.getOptionValue(ROUTE_OPTION, "default")); + + if (cl.hasOption(USER_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.USER); + builder.setId(cl.getOptionValue(USER_OPTION)); + } else if (cl.hasOption(GROUP_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.GROUP); + builder.setId(cl.getOptionValue(GROUP_OPTION)); + } else if (cl.hasOption(BUCKET_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.BUCKET); + builder.setId(cl.getOptionValue(BUCKET_OPTION)); + } else if (cl.hasOption(GID_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.GID); + builder.setId(cl.getOptionValue(GID_OPTION)); + builder.setDumpData(true); + } else if (cl.hasOption(DOCUMENT_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.DOCUMENT); + builder.setId(cl.getOptionValue(DOCUMENT_OPTION)); + builder.setDumpData(true); + } else if (!cl.hasOption(HELP_OPTION)) { + throw new IllegalArgumentException("Must specify one of 'user', 'group', 'bucket', 'document' or 'gid'."); + } + + return builder.build(); + } catch (ParseException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java new file mode 100644 index 00000000000..55a31f30a2b --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; + +/** + * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}. + * + * @author bjorncs + */ +public class DocumentAccessFactory { + public MessageBusDocumentAccess createDocumentAccess() { + return new MessageBusDocumentAccess(); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java new file mode 100644 index 00000000000..9d87a6f68f4 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * Main application class + * + * @author bjorncs + */ +public class Main { + + private Main() { + } + + public static void main(String[] args) { + CommandLineOptions options = new CommandLineOptions(); + try { + ClientParameters params = options.parseCommandLineArguments(args); + if (params.help) { + options.printHelp(); + return; + } + BucketStatsRetriever retriever = new BucketStatsRetriever( + new DocumentAccessFactory(), + params.route, + createShutdownHookRegistrar()); + BucketStatsPrinter printer = new BucketStatsPrinter(retriever, System.out); + printer.retrieveAndPrintBucketStats(params.selectionType, params.id, params.dumpData); + } catch (IllegalArgumentException e) { + System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage()); + } catch (BucketStatsException e) { + System.err.println(e.getMessage()); + } + } + + private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() { + return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable)); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java new file mode 100644 index 00000000000..803445d16f5 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java @@ -0,0 +1,162 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespasummarybenchmark; + +import com.yahoo.compress.CompressionType; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.jrt.*; +import com.yahoo.log.LogSetup; +import com.yahoo.slime.*; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * + * This is used for testing and benchmarking rpc docsum interface. + * time vespa-summary-benchmark file-containing-docids connectionspec summary-class repetitions threads + * fx ' time vespa-summary-benchmark feed.xml tcp/localhost:19115 keyvaluesummary 10000 32' + * + * @author baldersheim + */ +public class VespaSummaryBenchmark { + + private final Supervisor supervisor = new Supervisor(new Transport()); + + private VespaSummaryBenchmark() { } + + private static List<String> getDocIds(String fileName) { + try { + FileInputStream fstream = new FileInputStream(fileName); + DataInputStream in = new DataInputStream(fstream); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + + List<String> docIds = new ArrayList<>(); + while ((strLine = br.readLine()) != null) { + docIds.add(strLine); + } + in.close(); + return docIds; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private List<Target> getTargets(String connectionSpec, int numTargets) { + List<Target> targets = new ArrayList<>(numTargets); + for ( int i=0; i < numTargets; i++) { + targets.add(supervisor.connect(new Spec(connectionSpec))); + } + return targets; + } + + private static Slime createDocsumRequest(String summaryClass, List<GlobalId> gids) { + Slime docsumRequest = new Slime(); + Cursor root = docsumRequest.setObject(); + root.setString("class", summaryClass); + Cursor gidCursor = root.setArray("gids"); + for (GlobalId gid : gids) { + gidCursor.addData(gid.getRawId()); + } + return docsumRequest; + } + + private static class Waiter implements RequestWaiter { + + int waitingFor; + boolean dump; + + Waiter(int expect, boolean dump) { + waitingFor = expect; + this.dump = dump; + } + + private void print(Request request) { + Values ret = request.returnValues(); + CompressionType type = CompressionType.valueOf(ret.get(0).asInt8()); + int uncompressedSize = ret.get(1).asInt32(); + byte [] blob = ret.get(2).asData(); + if (type == CompressionType.LZ4) { + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + byte [] uncompressed = new byte [uncompressedSize]; + int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize); + if (compressedLength != blob.length) { + throw new DeserializationException("LZ4 decompression failed. compressed size does not match. Expected " + blob.length + ". Got " + compressedLength); + } + blob = uncompressed; + } + Slime slime = BinaryFormat.decode(blob); + try { + new JsonFormat(true).encode(System.out, slime); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void handleRequestDone(Request request) { + synchronized (this) { + if (dump) { + print(request); + dump = false; + } + waitingFor--; + if (waitingFor == 0) { + this.notifyAll(); + } + } + } + void waitForReplies() throws InterruptedException { + synchronized (this) { + while (waitingFor > 0) { + this.wait(); + } + } + } + } + + private static void fetchDocIds(String summaryClass, List<Target> targets, List<GlobalId> gids, boolean dump) { + Slime docsumRequest = createDocsumRequest(summaryClass, gids); + byte [] blob = BinaryFormat.encode(docsumRequest); + Waiter waiter = new Waiter(targets.size(), dump); + for (Target target : targets) { + Request r = new Request("proton.getDocsums"); + r.parameters().add(new Int8Value(CompressionType.NONE.getCode())); + r.parameters().add(new Int32Value(blob.length)); + r.parameters().add(new DataValue(blob)); + target.invokeAsync(r, 100.0, waiter); + } + try { + waiter.waitForReplies(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + public static void main(String[] args) { + LogSetup.initVespaLogging("vespasummarybenchmark"); + String docidFileName = args[0]; + String connectionSpec = args[1]; + String summaryClass = args[2]; + int numRuns = Integer.parseInt(args[3]); + int numTargets = Integer.parseInt(args[4]); + VespaSummaryBenchmark benchmark = new VespaSummaryBenchmark(); + List<String> docIds = getDocIds(docidFileName); + List<GlobalId> gids = new ArrayList<>(docIds.size()); + for (String docid : docIds) { + GlobalId gid = new GlobalId(IdString.createIdString(docid)); + gids.add(gid); + } + List<Target> targets = benchmark.getTargets(connectionSpec, numTargets); + for (int i = 0; i < numRuns; i++) { + fetchDocIds(summaryClass, targets, gids, i==0); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java new file mode 100644 index 00000000000..c849bc1741c --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -0,0 +1,292 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.documentapi.AckToken; +import com.yahoo.documentapi.DumpVisitorDataHandler; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; +import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; +import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; +import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Message; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * A visitor data and progress handler that writes to STDOUT. + * + * Due to java not being able to inherit two classes, and neither being an + * interface this had to be implemented by creating a wrapper class. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class StdOutVisitorHandler extends VdsVisitHandler { + private static final Logger log = Logger.getLogger( + StdOutVisitorHandler.class.getName()); + private boolean printIds; + private boolean indentXml; + private int processTimeMilliSecs; + private PrintStream out; + private final boolean jsonOutput; + + private VisitorDataHandler dataHandler; + + public StdOutVisitorHandler(boolean printIds, boolean indentXml, + boolean showProgress, boolean showStatistics, boolean doStatistics, + boolean abortOnClusterDown, int processtime, boolean jsonOutput) + { + super(showProgress, showStatistics, abortOnClusterDown); + + this.printIds = printIds; + this.indentXml = indentXml; + this.processTimeMilliSecs = processtime; + this.jsonOutput = jsonOutput; + String charset = "UTF-8"; + try { + out = new PrintStream(System.out, true, charset); + } catch (java.io.UnsupportedEncodingException e) { + System.out.println(charset + " is an unsupported encoding, " + + "using default instead."); + out = System.out; + } + + dataHandler = new DataHandler(doStatistics); + } + + @Override + public void onDone() { + } + + public VisitorDataHandler getDataHandler() { return dataHandler; } + + class StatisticsMap extends LinkedHashMap<String, Integer> { + int maxSize; + + StatisticsMap(int maxSize) { + super(100, (float)0.75, true); + this.maxSize = maxSize; + } + + protected boolean removeEldestEntry(Map.Entry<String, Integer> eldest) { + if (size() > maxSize) { + dump(eldest); + return true; + } + + return false; + } + + private void dump(Map.Entry<String, Integer> e) { + out.println(e.getKey() + ":" + e.getValue()); + } + + public void dumpAll() { + for (Map.Entry<String, Integer> e : entrySet()) { + dump(e); + } + clear(); + } + } + + class DataHandler extends DumpVisitorDataHandler { + boolean doStatistics; + StatisticsMap statisticsMap = new StatisticsMap(10000); + private volatile boolean first = true; + + public DataHandler(boolean doStatistics) { + this.doStatistics = doStatistics; + } + + @Override + public void onMessage(Message m, AckToken token) { + if (processTimeMilliSecs > 0) { + try { + Thread.sleep(processTimeMilliSecs); + } catch (InterruptedException e) {} + } + + synchronized (printLock) { + if (m instanceof MapVisitorMessage) { + onMapVisitorData(((MapVisitorMessage)m).getData()); + ack(token); + } else if (m instanceof DocumentListMessage) { + DocumentListMessage dlm = (DocumentListMessage)m; + onDocumentList(dlm.getBucketId(), dlm.getDocuments()); + ack(token); + } else if (m instanceof EmptyBucketsMessage) { + onEmptyBuckets(((EmptyBucketsMessage)m).getBucketIds()); + ack(token); + } else { + super.onMessage(m, token); + } + } + } + + @Override + public void onDocument(Document doc, long timestamp) { + try { + if (lastLineIsProgress) { + System.err.print('\r'); + } + + if (printIds) { + out.print(doc.getId()); + out.print(" (Last modified at "); + out.println(timestamp + ")"); + } else { + if (jsonOutput) { + writeJsonDocument(doc); + } else { + out.print(doc.toXML( + indentXml ? " " : "")); + } + } + } catch (Exception e) { + System.err.println("Failed to output document: " + + e.getMessage()); + getControlHandler().abort(); + } + } + + private void writeJsonDocument(Document doc) throws IOException { + writeFeedStartOrRecordSeparator(); + out.write(JsonWriter.toByteArray(doc)); + } + + @Override + public void onRemove(DocumentId docId) { + try { + if (lastLineIsProgress) { + System.err.print('\r'); + } + + if (printIds) { + out.println(docId + " (Removed)"); + } else { + if (jsonOutput) { + writeJsonDocumentRemove(docId); + } else { + XmlStream stream = new XmlStream(); + stream.beginTag("remove"); + stream.addAttribute("documentid", docId); + stream.endTag(); + assert(stream.isFinalized()); + out.print(stream); + } + } + } catch (Exception e) { + System.err.println("Failed to output document: " + + e.getMessage()); + getControlHandler().abort(); + } + } + + private void writeJsonDocumentRemove(DocumentId docId) + throws IOException { + writeFeedStartOrRecordSeparator(); + out.write(JsonWriter.documentRemove(docId)); + } + + private void writeFeedStartOrRecordSeparator() { + if (first) { + out.println("["); + first = false; + } else { + out.println(","); + } + } + + private void writeFeedEnd() { + out.println("]"); + } + + public void onMapVisitorData(Map<String, String> data) { + for (String key : data.keySet()) { + if (doStatistics) { + Integer i = statisticsMap.get(key); + if (i != null) { + statisticsMap.put(key, Integer.parseInt(data.get(key)) + i); + } else { + statisticsMap.put(key, Integer.parseInt(data.get(key))); + } + } else { + out.println(key + ":" + data.get(key)); + } + } + } + + public void onDocumentList(BucketId bucketId, List<DocumentListEntry> documents) { + out.println("Got document list of bucket " + bucketId.toString()); + for (DocumentListEntry entry : documents) { + entry.getDocument().setLastModified(entry.getTimestamp()); + onDocument(entry.getDocument(), entry.getTimestamp()); + } + } + + public void onEmptyBuckets(List<BucketId> bucketIds) { + StringBuilder buckets = new StringBuilder(); + for(BucketId bid : bucketIds) { + buckets.append(" "); + buckets.append(bid.toString()); + } + log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets); + } + + public synchronized void onDone() { + if (jsonOutput) { + writeFeedEnd(); + } + statisticsMap.dumpAll(); + super.onDone(); + } + } + + class ControlHandler extends VisitorControlHandler { + public void onProgress(ProgressToken token) { + if (showProgress) { + synchronized (printLock) { + if (lastLineIsProgress) { + System.err.print('\r'); + } + System.err.format("%.1f %% finished.", + token.percentFinished()); + lastLineIsProgress = true; + } + } + super.onProgress(token); + } + + public void onDone(CompletionCode code, String message) { + if (lastLineIsProgress) { + System.err.print('\n'); + lastLineIsProgress = false; + } + if (code != CompletionCode.SUCCESS) { + if (code == CompletionCode.ABORTED) { + System.err.println("Visitor aborted: " + message); + } else if (code == CompletionCode.TIMEOUT) { + System.err.println("Visitor timed out: " + message); + } else { + System.err.println("Visitor aborted due to unknown issue " + + code + ": " + message); + } + } else if (showProgress) { + System.err.println("Completed visiting."); + } + super.onDone(code, message); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java new file mode 100644 index 00000000000..ff072b845de --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -0,0 +1,789 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.log.LogSetup; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.Map; + +/** + * Example client using visiting + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class VdsVisit { + private VdsVisitParameters params; + private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet()); + private VisitorSession session; + + private final VisitorSessionAccessorFactory sessionAccessorFactory; + private VisitorSessionAccessor sessionAccessor; + private ShutdownHookRegistrar shutdownHookRegistrar; + + public interface ShutdownHookRegistrar { + public void registerShutdownHook(Thread thread); + } + + public interface VisitorSessionAccessor { + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; + public void shutdown(); + } + + public interface VisitorSessionAccessorFactory { + public VisitorSessionAccessor createVisitorSessionAccessor(); + } + + private static class MessageBusVisitorSessionAccessor implements VisitorSessionAccessor { + private MessageBusDocumentAccess access; + + private MessageBusVisitorSessionAccessor(MessageBusParams mbparams) { + access = new MessageBusDocumentAccess(mbparams); + } + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return access.createVisitorSession(params); + } + + @Override + public void shutdown() { + access.shutdown(); + } + } + + private static class MessageBusVisitorSessionAccessorFactory implements VisitorSessionAccessorFactory { + MessageBusParams mbparams; + + private MessageBusVisitorSessionAccessorFactory(MessageBusParams mbparams) { + this.mbparams = mbparams; + } + + @Override + public VisitorSessionAccessor createVisitorSessionAccessor() { + return new MessageBusVisitorSessionAccessor(mbparams); + } + } + + private static class JvmRuntimeShutdownHookRegistrar implements ShutdownHookRegistrar { + @Override + public void registerShutdownHook(Thread thread) { + Runtime.getRuntime().addShutdownHook(thread); + } + } + + public VdsVisit() { + this.sessionAccessorFactory = new MessageBusVisitorSessionAccessorFactory(mbparams); + this.shutdownHookRegistrar = new JvmRuntimeShutdownHookRegistrar(); + } + + public VdsVisit(VisitorSessionAccessorFactory sessionAccessorFactory, + ShutdownHookRegistrar shutdownHookRegistrar) + { + this.sessionAccessorFactory = sessionAccessorFactory; + this.shutdownHookRegistrar = shutdownHookRegistrar; + } + + public static void main(String args[]) { + LogSetup.initVespaLogging("vespavisit"); + VdsVisit vdsVisit = new VdsVisit(); + + Options options = createOptions(); + + try { + ArgumentParser parser = new ArgumentParser(options); + vdsVisit.params = parser.parse(args); + if (vdsVisit.params == null) { + vdsVisit.printSyntax(options); + System.exit(0); + } + ClusterList clusterList = new ClusterList("client"); + vdsVisit.params.getVisitorParameters().setRoute( + resolveClusterRoute(clusterList, vdsVisit.params.getCluster())); + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage()); + System.exit(1); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + } + + if (vdsVisit.params.isVerbose()) { + verbosePrintParameters(vdsVisit.params, System.err); + } + + try { + vdsVisit.run(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private void printSyntax(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vespavisit <options>", "Visit documents from VDS", options , ""); + } + + @SuppressWarnings("AccessStaticViaInstance") + protected static Options createOptions() { + Options options = new Options(); + options.addOption("h", "help", false, "Show this syntax page."); + + options.addOption(Option.builder("d") + .longOpt("datahandler") + .hasArg(true) + .argName("target") + .desc("Send results to the given target.") + .build()); + + options.addOption(Option.builder("s") + .longOpt("selection") + .hasArg(true) + .argName("selection") + .desc("What documents to visit.") + .build()); + + options.addOption(Option.builder("f") + .longOpt("from") + .hasArg(true) + .argName("timestamp") + .desc("Only visit from the given timestamp (microseconds).") + .type(Number.class) + .build()); + + options.addOption(Option.builder("t") + .longOpt("to") + .hasArg(true) + .argName("timestamp") + .desc("Only visit up to the given timestamp (microseconds).") + .type(Number.class).build()); + + options.addOption("e", "headersonly", false, "Only visit headers of documents.[Deprecated]"); + + options.addOption(Option.builder("l") + .longOpt("fieldset") + .hasArg(true) + .argName("fieldset") + .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html). Default is [all].") + .build()); + + options.addOption(Option.builder() + .longOpt("visitinconsistentbuckets") + .hasArg(false) + .desc("Don't wait for inconsistent buckets to become consistent.") + .build()); + + options.addOption(Option.builder("m") + .longOpt("maxpending") + .hasArg(true) + .argName("num") + .desc("Maximum pending messages to data handlers per storage visitor.") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("maxpendingsuperbuckets") + .hasArg(true) + .argName("num") + .desc("Maximum pending visitor messages from the vespavisit client. If set, dynamic throttling of visitors will be disabled!") + .type(Number.class) + .build()); + + options.addOption(Option.builder("b") + .longOpt("maxbuckets") + .hasArg(true) + .argName("num") + .desc("Maximum buckets per visitor.") + .type(Number.class) + .build()); + + options.addOption("i", "printids", false, "Display only document identifiers."); + + options.addOption(Option.builder("p") + .longOpt("progress") + .hasArg(true) + .argName("file") + .desc("Use given file to track progress.") + .build()); + + options.addOption(Option.builder("o") + .longOpt("timeout") + .hasArg(true) + .argName("milliseconds") + .desc("Time out visitor after given time.") + .type(Number.class) + .build()); + + options.addOption(Option.builder("u") + .longOpt("buckettimeout") + .hasArg(true) + .argName("milliseconds") + .desc("Fail visitor if visiting a single bucket takes longer than this (default same as timeout)") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("visitlibrary") + .hasArg(true) + .argName("string") + .desc("Use the given visitor library.") + .build()); + + options.addOption(Option.builder() + .longOpt("libraryparam") + .numberOfArgs(2) + .argName("key> <val") + .desc("Give the following parameter to the visitor.") + .build()); + + options.addOption("r", "visitremoves", false, "Include information of removed documents."); + + options.addOption(Option.builder("c") + .longOpt("cluster") + .hasArg(true) + .argName("cluster") + .desc("Visit the given VDS cluster.") + .build()); + + options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR."); + + options.addOption(Option.builder() + .longOpt("statistics") + .hasArg(true) + .argName("args") + .desc("Use CountVisitor for document statistics. Use comma-separated arguments.") + .build()); + + options.addOption(Option.builder() + .longOpt("abortonclusterdown") + .hasArg(false) + .desc("Abort if cluster is down.") + .build()); + + options.addOption(Option.builder() + .longOpt("maxhits") + .hasArg(true) + .argName("num") + .desc("Abort visiting when we have received this many \"first pass\" documents. Only appropriate for visiting involving id.order. This is only an approximate number, all pending work will be completed and those documents will also be returned.") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("maxtotalhits") + .hasArg(true) + .argName("num") + .desc("Abort visiting when we have received this many total documents. This is only an approximate number, all pending work will be completed and those documents will also be returned.") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("processtime") + .hasArg(true) + .argName("num") + .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client)") + .type(Number.class) + .build()); + options.addOption(Option.builder() + .longOpt("priority") + .hasArg(true) + .argName("name") + .desc("Priority used for each visitor. Defaults to NORMAL_3. " + + "Use with care to avoid starving lower prioritized traffic in the cluster") + .build()); + + options.addOption(Option.builder() + .longOpt("ordering") + .hasArg(true) + .argName("order") + .desc("Order to visit documents in. Only makes sense in conjunction with a document selection involving id.order. Legal values are \"ascending\" and \"descending\"") + .build()); + + options.addOption(Option.builder() + .longOpt("tracelevel") + .hasArg(true) + .argName("level") + .desc("Tracelevel ([0-9]) to use for debugging purposes") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("skipbucketsonfatalerrors") + .hasArg(false) + .desc("Skip visiting super buckets with fatal error codes.") + .build()); + + options.addOption(Option.builder() + .longOpt("jsonoutput") + .desc("Output documents as JSON") + .hasArg(false) + .build()); + return options; + } + + public static class VdsVisitParameters { + private VisitorParameters visitorParameters; + /** If not specified in options, will get form cluster list */ + private String cluster = null; + private boolean verbose = false; + private boolean printIdsOnly = false; + private String statisticsParts = null; + private boolean abortOnClusterDown = false; + private int processTime = 0; + private int fullTimeout = 7 * 24 * 60 * 60 * 1000; + private boolean jsonOutput = false; + + public VisitorParameters getVisitorParameters() { + return visitorParameters; + } + + public void setVisitorParameters(VisitorParameters visitorParameters) { + this.visitorParameters = visitorParameters; + } + + public String getCluster() { + return cluster; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } + + public boolean isVerbose() { + return verbose; + } + + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + public boolean isPrintIdsOnly() { + return printIdsOnly; + } + + public void setPrintIdsOnly(boolean printIdsOnly) { + this.printIdsOnly = printIdsOnly; + } + + public String getStatisticsParts() { + return statisticsParts; + } + + public void setStatisticsParts(String statisticsParts) { + this.statisticsParts = statisticsParts; + } + + public boolean getAbortOnClusterDown() { + return abortOnClusterDown; + } + + public void setAbortOnClusterDown(boolean abortOnClusterDown) { + this.abortOnClusterDown = abortOnClusterDown; + } + + public int getFullTimeout() { + return fullTimeout; + } + + public void setFullTimeout(int fullTimeout) { + this.fullTimeout = fullTimeout; + } + + public int getProcessTime() { + return processTime; + } + + public void setProcessTime(int processTime) { + this.processTime = processTime; + } + + public void setJsonOutput(boolean jsonOutput) { + this.jsonOutput = jsonOutput; + } + } + + protected static class ArgumentParser { + private Options options; + + public ArgumentParser(Options options) { + this.options = options; + } + + public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException { + VdsVisitParameters allParams = new VdsVisitParameters(); + VisitorParameters params = new VisitorParameters(""); + CommandLineParser parser = new DefaultParser(); + CommandLine line = parser.parse(options, args); + + if (line.hasOption("h")) { + return null; + } + if (line.hasOption("d")) { + params.setRemoteDataHandler(line.getOptionValue("d")); + } + if (line.hasOption("s")) { + params.setDocumentSelection(line.getOptionValue("s")); + } + if (line.hasOption("f")) { + params.setFromTimestamp(((Number) line.getParsedOptionValue("f")).longValue()); + } + if (line.hasOption("t")) { + params.setToTimestamp(((Number) line.getParsedOptionValue("t")).longValue()); + } + if (line.hasOption("e")) { + params.fieldSet("[header]"); + } + if (line.hasOption("l")) { + params.fieldSet(line.getOptionValue("l")); + } + if (line.hasOption("visitinconsistentbuckets")) { + params.visitInconsistentBuckets(true); + } + if (line.hasOption("m")) { + params.setMaxPending(((Number) line.getParsedOptionValue("m")).intValue()); + } + if (line.hasOption("b")) { + params.setMaxBucketsPerVisitor(((Number) line.getParsedOptionValue("b")).intValue()); + } + if (line.hasOption("i")) { + allParams.setPrintIdsOnly(true); + params.fieldSet("[id]"); + } + if (line.hasOption("p")) { + params.setResumeFileName(line.getOptionValue("p")); + } + if (line.hasOption("o")) { + allParams.setFullTimeout(((Number) line.getParsedOptionValue("o")).intValue()); + params.setTimeoutMs(allParams.getFullTimeout()); + } + if (line.hasOption("u")) { + params.setTimeoutMs(((Number) line.getParsedOptionValue("u")).intValue()); + } + if (line.hasOption("visitlibrary")) { + params.setVisitorLibrary(line.getOptionValue("visitlibrary")); + } + if (line.hasOption("libraryparam")) { + String key = line.getOptionValues("libraryparam")[0]; + String value = line.getOptionValues("libraryparam")[1]; + params.setLibraryParameter(key, value); + } + if (line.hasOption("r")) { + params.visitRemoves(true); + } + if (line.hasOption("c")) { + allParams.setCluster(line.getOptionValue("c")); + } + + if (line.hasOption("v")) { + allParams.setVerbose(true); + } + + if (line.hasOption("statistics")) { + allParams.setStatisticsParts(line.getOptionValue("statistics")); + params.fieldSet("[id]"); + params.setVisitorLibrary("CountVisitor"); + } + + if (line.hasOption("abortonclusterdown")) { + allParams.setAbortOnClusterDown(true); + } + if (line.hasOption("processtime")) { + allParams.setProcessTime(((Number) line.getParsedOptionValue("processtime")).intValue()); + } + if (line.hasOption("maxhits")) { + params.setMaxFirstPassHits(((Number)line.getParsedOptionValue("maxhits")).intValue()); + } + if (line.hasOption("maxtotalhits")) { + params.setMaxTotalHits(((Number)line.getParsedOptionValue("maxtotalhits")).intValue()); + } + if (line.hasOption("tracelevel")) { + params.setTraceLevel(((Number)line.getParsedOptionValue("tracelevel")).intValue()); + } + if (line.hasOption("priority")) { + try { + DocumentProtocol.Priority priority = DocumentProtocol.getPriorityByName( + line.getOptionValue("priority")); + params.setPriority(priority); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown priority name"); + } + } else { + // Let bulk visitor jobs have a low priority by default to avoid stalling concurrent + // (real time) write and read operations. + params.setPriority(DocumentProtocol.Priority.LOW_1); + } + if (line.hasOption("ordering")) { + String opt = line.getOptionValue("ordering"); + if (opt.equalsIgnoreCase("ascending")) { + params.setVisitorOrdering(OrderingSpecification.ASCENDING); + } else if (opt.equalsIgnoreCase("descending")) { + params.setVisitorOrdering(OrderingSpecification.DESCENDING); + } else { + throw new IllegalArgumentException("Unknown ordering. Legal values are \"ascending\", \"descending\""); + } + } + if (line.hasOption("skipbucketsonfatalerrors")) { + params.skipBucketsOnFatalErrors(true); + } + if (line.hasOption("maxpendingsuperbuckets")) { + StaticThrottlePolicy throttlePolicy = new StaticThrottlePolicy(); + throttlePolicy.setMaxPendingCount(((Number)line.getParsedOptionValue("maxpendingsuperbuckets")).intValue()); + params.setThrottlePolicy(throttlePolicy); + } + if (line.hasOption("jsonoutput")) { + allParams.setJsonOutput(true); + } + + allParams.setVisitorParameters(params); + return allParams; + } + } + + // For unit testing only + protected void setVdsVisitParameters(VdsVisitParameters vdsVisitParameters) { + this.params = vdsVisitParameters; + } + + protected static String resolveClusterRoute(ClusterList clusters, String wantedCluster) { + if (clusters.getStorageClusters().size() == 0) { + throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " + + "declared. Visiting feature is not available."); + } + + ClusterDef found = null; + + String names = ""; + for (ClusterDef c : clusters.getStorageClusters()) { + if (!names.isEmpty()) { + names += ", "; + } + names += c.getName(); + } + if (wantedCluster != null) { + for (ClusterDef c : clusters.getStorageClusters()) { + if (c.getName().equals(wantedCluster)) { + found = c; + } + } + if (found == null) { + throw new IllegalArgumentException("Your vespa cluster contains the content clusters " + + names + ", not " + wantedCluster + ". Please select a valid vespa cluster."); + } + } else if (clusters.getStorageClusters().size() == 1) { + found = clusters.getStorageClusters().get(0); + } else { + throw new IllegalArgumentException("Your vespa cluster contains the content clusters " + + names + ". Please use the -c option to select one of them as a target for visiting."); + } + + return "[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]"; + } + + protected static void verbosePrintParameters(VdsVisitParameters vdsParams, PrintStream out) { + VisitorParameters params = vdsParams.getVisitorParameters(); + if (params.getTimeoutMs() != -1) { + out.println("Time out visitor after " + params.getTimeoutMs() + " ms."); + } + if (params.getDocumentSelection() == null || params.getDocumentSelection().equals("")) { + out.println("Visiting all documents"); + } else { + out.println("Visiting documents matching: " + params.getDocumentSelection()); + } + if (params.getFromTimestamp() != 0 && params.getToTimestamp() != 0) { + out.println("Visiting in the inclusive timestamp range " + + params.getFromTimestamp() + " - " + params.getToTimestamp() + "."); + } else if (params.getFromTimestamp() != 0) { + out.println("Visiting from and including timestamp " + params.getFromTimestamp() + "."); + } else if (params.getToTimestamp() != 0) { + out.println("Visiting to and including timestamp " + params.getToTimestamp() + "."); + } + out.println("Visiting field set " + params.fieldSet() + "."); + if (params.visitInconsistentBuckets()) { + out.println("Visiting inconsistent buckets."); + } + if (params.visitRemoves()) { + out.println("Including remove entries."); + } + if (params.getResumeFileName() != null && !"".equals(params.getResumeFileName())) { + out.println("Tracking progress in file: " + params.getResumeFileName()); + } + if (vdsParams.isPrintIdsOnly()) { + out.println("Only showing document identifiers."); + } + out.println("Let visitor have maximum " + params.getMaxPending() + " replies pending on data handlers per storage node visitor."); + out.println("Visit maximum " + params.getMaxBucketsPerVisitor() + " buckets per visitor."); + if (params.getRemoteDataHandler() != null) { + out.println("Sending data to data handler at: " + params.getRemoteDataHandler()); + } + if (params.getRoute() != null) { + out.println("Visiting cluster '" + params.getRoute() + "'."); + } + if (params.getVisitorLibrary() != null) { + out.println("Using visitor library '" + params.getVisitorLibrary() + "'."); + } + if (params.getLibraryParameters().size() > 0) { + out.println("Adding the following library specific parameters:"); + for (Map.Entry<String, byte[]> entry : params.getLibraryParameters().entrySet()) { + out.println(" " + entry.getKey() + " = " + + new String(entry.getValue(), Charset.forName("utf-8"))); + } + } + if (params.getPriority() != DocumentProtocol.Priority.NORMAL_3) { + out.println("Visitor priority " + params.getPriority().name()); + } + if (params.skipBucketsOnFatalErrors()) { + out.println("Skip visiting super buckets with fatal errors."); + } + } + + private void onDocumentSelectionException(Exception e) { + System.err.println("Illegal document selection string '" + + params.getVisitorParameters().getDocumentSelection() + "'.\n"); + System.exit(1); + } + + private void onIllegalArgumentException(Exception e) { + System.err.println("Illegal arguments : \n"); + System.err.println(e.getMessage()); + System.exit(1); + } + + public void run() { + System.exit(doRun()); + } + + protected int doRun() { + VisitorParameters visitorParameters = params.getVisitorParameters(); + // If progress file already exists, create resume token from it + if (visitorParameters.getResumeFileName() != null && + !"".equals(visitorParameters.getResumeFileName())) + { + try { + File file = new File(visitorParameters.getResumeFileName()); + FileInputStream fos = new FileInputStream(file); + + StringBuilder builder = new StringBuilder(); + byte[] b = new byte[100000]; + int length; + + while ((length = fos.read(b)) > 0) { + builder.append(new String(b, 0, length)); + } + fos.close(); + visitorParameters.setResumeToken(new ProgressToken(builder.toString())); + + if (params.isVerbose()) { + System.err.format("Resuming visitor already %.1f %% finished.\n", + visitorParameters.getResumeToken().percentFinished()); + } + } catch (FileNotFoundException e) { + // Ignore; file has not been created yet but will be shortly. + } catch (IOException e) { + System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName()); + e.printStackTrace(System.err); + return 1; + } + } + + initShutdownHook(); + sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor(); + + VdsVisitHandler handler; + + handler = new StdOutVisitorHandler( + params.isPrintIdsOnly(), + params.isVerbose(), + params.isVerbose(), + params.isVerbose(), + params.getStatisticsParts() != null, + params.getAbortOnClusterDown(), + params.getProcessTime(), + params.jsonOutput); + + if (visitorParameters.getResumeFileName() != null) { + handler.setProgressFileName(visitorParameters.getResumeFileName()); + } + + visitorParameters.setControlHandler(handler.getControlHandler()); + if (visitorParameters.getRemoteDataHandler() == null) { + visitorParameters.setLocalDataHandler(handler.getDataHandler()); + } + + if (params.getStatisticsParts() != null) { + String[] parts = params.getStatisticsParts().split(","); + for (String s : parts) { + visitorParameters.setLibraryParameter(s, "true"); + } + } + + try { + session = sessionAccessor.createVisitorSession(visitorParameters); + while (true) { + try { + if (session.waitUntilDone(params.getFullTimeout())) break; + } catch (InterruptedException e) {} + } + + if (visitorParameters.getTraceLevel() > 0) { + System.out.println(session.getTrace().toString()); + } + } catch (ParseException e) { + onDocumentSelectionException(e); + } catch (IllegalArgumentException e) { + onIllegalArgumentException(e); + } catch (Exception e) { + System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection()); + System.err.println("Caught unexpected exception: "); + e.printStackTrace(System.err); + return 1; + } + if (visitorParameters.getControlHandler().getResult().code + == VisitorControlHandler.CompletionCode.SUCCESS) + { + return 0; + } else { + return 1; + } + } + + private void initShutdownHook() { + shutdownHookRegistrar.registerShutdownHook(new CleanUpThread()); + } + + class CleanUpThread extends Thread { + public void run() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException ise) { + //ignore this + } + try { + if (sessionAccessor != null) { + sessionAccessor.shutdown(); + } + } catch (IllegalStateException ise) { + //ignore this too + } + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java new file mode 100644 index 00000000000..ad3e0a0b359 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java @@ -0,0 +1,181 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.vdslib.VisitorStatistics; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import java.util.Date; +import java.util.TimeZone; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; + +/** + * An abstract class that can be subclassed by different visitor handlers. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public abstract class VdsVisitHandler { + boolean showProgress; + boolean showStatistics; + boolean abortOnClusterDown; + boolean lastLineIsProgress = false; + String lastPercentage; + final Object printLock = new Object(); + + protected String progressFileName = ""; + + final VisitorControlHandler controlHandler = new ControlHandler(); + + public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) + { + this.showProgress = showProgress; + this.showStatistics = showStatistics; + this.abortOnClusterDown = abortOnClusterDown; + } + + public boolean getShowProgress() { + return showProgress; + } + + public boolean getShowStatistics() { + return showStatistics; + } + + public boolean getAbortOnClusterDown() { + return abortOnClusterDown; + } + + public boolean getLastLineIsProgress() { + return lastLineIsProgress; + } + + public void setLastLineIsProgress(boolean isProgress) { + lastLineIsProgress = isProgress; + } + + public String getLastPercentage() { + return lastPercentage; + } + + public void setLastPercentage(String lastPercentage) { + this.lastPercentage = lastPercentage; + } + + public Object getPrintLock() { + return printLock; + } + + public void onDone() { + } + + public String getProgressFileName() { + return progressFileName; + } + + public void setProgressFileName(String progressFileName) { + this.progressFileName = progressFileName; + } + + public VisitorControlHandler getControlHandler() { return controlHandler; } + public abstract VisitorDataHandler getDataHandler(); + + class ControlHandler extends VisitorControlHandler { + VisitorStatistics statistics; + + public void onProgress(ProgressToken token) { + if (progressFileName.length() > 0) { + try { + synchronized (token) { + File file = new File(progressFileName + ".tmp"); + FileOutputStream fos = new FileOutputStream(file); + fos.write(token.toString().getBytes()); + fos.close(); + file.renameTo(new File(progressFileName)); + } + } + catch (IOException e) { + e.printStackTrace(); + } + } + if (showProgress) { + synchronized (printLock) { + DecimalFormat df = new DecimalFormat("#.#"); + String percentage = df.format(token.percentFinished()); + if (!percentage.equals(lastPercentage)) { + if (lastLineIsProgress) { + System.err.print('\r'); + } + System.err.print(percentage + " % finished."); + lastLineIsProgress = true; + lastPercentage = percentage; + } + } + } + super.onProgress(token); + } + + @Override + public void onVisitorStatistics(VisitorStatistics visitorStatistics) { + statistics = visitorStatistics; + } + + private String getDateTime() { + DateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = new Date(); + return dateFormat.format(date); + } + + public void onVisitorError(String message) { + synchronized (printLock) { + if (lastLineIsProgress) { + System.err.print('\r'); + lastLineIsProgress = false; + } + System.err.println("Visitor error (" + getDateTime() + "): " + + message); + if (abortOnClusterDown && + !isDone() && + (message.lastIndexOf("Could not resolve")>=0 || + message.lastIndexOf("don't allow external load")>=0)) { + System.err.println("Aborting visitor as " + + "--abortonclusterdown flag is set."); + abort(); + } + } + } + public void onDone(CompletionCode code, String message) { + if (lastLineIsProgress) { + System.err.print('\n'); + lastLineIsProgress = false; + } + if (code != CompletionCode.SUCCESS) { + if (code == CompletionCode.ABORTED) { + System.err.println("Visitor aborted: " + message); + } else if (code == CompletionCode.TIMEOUT) { + System.err.println("Visitor timed out: " + message); + } else { + System.err.println("Visitor aborted due to unknown issue " + + code + ": " + message); + } + } else { + if (showProgress) { + System.err.println("Completed visiting."); + } + if (showStatistics) { + System.err.println("*** Visitor statistics"); + System.err.println(statistics == null ? "Nothing visited" : statistics.toString()); + } + } + super.onDone(code, message); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java new file mode 100644 index 00000000000..3ef0619cfd8 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -0,0 +1,286 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogSetup; +import com.yahoo.messagebus.network.Identity; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.lang.reflect.Constructor; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.logging.Logger; + +/** + * Example client using visiting + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class VdsVisitTarget { + private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName()); + + private boolean printIds = false; + DocumentAccess access; + VisitorDestinationSession session; + String slobrokAddress = null; + int port = -1; + private boolean verbose = false; + private int processTime = 0; + private String handlerClassName = StdOutVisitorHandler.class.getName(); + private String[] handlerArgs = null; + + public boolean isPrintIds() { + return printIds; + } + + public String getSlobrokAddress() { + return slobrokAddress; + } + + public boolean isVerbose() { + return verbose; + } + + public int getPort() { + return port; + } + + public int getProcessTime() { + return processTime; + } + + public String getHandlerClassName() { + return handlerClassName; + } + + public String[] getHandlerArgs() { + return handlerArgs; + } + + public static void main(String args[]) { + LogSetup.initVespaLogging("vespavisittarget"); + VdsVisitTarget visitTarget = new VdsVisitTarget(); + + + try { + visitTarget.parseArguments(args); + visitTarget.initShutdownHook(); + visitTarget.run(); + System.exit(0); + } catch (HelpShownException e) { + System.exit(0); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage()); + System.exit(1); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static Options createOptions() { + Options options = new Options(); + + options.addOption("h", "help", false, "Show this syntax page."); + + options.addOption(Option.builder("s") + .longOpt("bindtoslobrok") + .hasArg(true) + .argName("address") + .desc("Bind to the given slobrok address.") + .build()); + + options.addOption(Option.builder("t") + .longOpt("bindtosocket") + .hasArg(true) + .argName("port") + .desc("Bind to the given TCP port") + .type(Number.class) + .build()); + + options.addOption(Option.builder("p") + .longOpt("processtime") + .hasArg(true) + .argName("msecs") + .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client).") + .type(Number.class) + .build()); + + options.addOption(Option.builder("c") + .longOpt("visithandler") + .hasArg(true) + .argName("classname") + .desc("Use the given class as a visit handler (defaults to StdOutVisitorHandler)") + .build()); + + options.addOption(Option.builder("o") + .longOpt("visitoptions") + .hasArg(true) + .argName("args") + .desc("Option arguments to pass through to the visitor handler instance") + .build()); + + options.addOption("i", "printids", false, "Display only document identifiers."); + options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR."); + + return options; + } + + private void printSyntax(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vespavisittarget <options>", "Retrieve results from a visitor", options , + "One, and only one, of the binding options must be present.\n" + + "\n" + + "For more detailed information, such as defaults and format of\n" + + "arguments, refer to 'man vespavisittarget'.\n"); + } + + class HelpShownException extends Exception {} + + void parseArguments(String args[]) throws ParseException, HelpShownException { + Options options = createOptions(); + + CommandLineParser parser = new DefaultParser(); + CommandLine line = parser.parse(options, args); + + if (line.hasOption("h")) { + printSyntax(options); + throw new HelpShownException(); + } + if (line.hasOption("s")) { + slobrokAddress = line.getOptionValue("s"); + } + if (line.hasOption("t")) { + port = ((Number) line.getParsedOptionValue("t")).intValue(); + } + if (line.hasOption("i")) { + printIds = true; + } + if (line.hasOption("p")) { + processTime = ((Number) line.getParsedOptionValue("p")).intValue(); + } + if (line.hasOption("v")) { + verbose = true; + } + if (line.hasOption("c")) { + handlerClassName = line.getOptionValue("c"); + } + if (line.hasOption("o")) { + handlerArgs = line.getOptionValue("o").split(" "); + } + + if (!(slobrokAddress == null ^ port == -1)) { + throw new IllegalArgumentException("You must specify one, and only one, binding option"); + } + if (port != -1 && port < 0 || port > 65535) { + throw new IllegalArgumentException("The port must be in the range 0-65535"); + } + if (verbose) { + if (port != -1) { + System.err.println("Binding to socket " + getTcpAddress()); + } else { + System.err.println("Binding to slobrok address: " + slobrokAddress + "/visit-destination"); + } + } + } + + private String getTcpAddress() { + try { + InetAddress addr = InetAddress.getLocalHost(); + String hostname = addr.getHostName(); + return "tcp/" + hostname + ":" + port + "/visit-destination"; + } catch (UnknownHostException e) { + System.err.println("Failed to detect hostname."); + System.exit(1); + } + return ""; + } + + @SuppressWarnings("unchecked") + public void run() throws Exception { + initShutdownHook(); + log.log(LogLevel.DEBUG, "Starting VdsVisitTarget"); + MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); + mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); + + if (port > 0) { + mbusParams.getRPCNetworkParams().setListenPort(port); + } + + access = new MessageBusDocumentAccess(mbusParams); + + VdsVisitHandler handler; + + Class<?> cls = Thread.currentThread().getContextClassLoader() + .loadClass(handlerClassName); + try { + // Any custom data handlers may have a constructor that takes in args, + // so that the user can pass cmd line options to them + Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class, + boolean.class, boolean.class, boolean.class, + boolean.class, int.class, String[].class }; + Constructor<?> cons = cls.getConstructor(consTypes); + handler = (VdsVisitHandler)cons.newInstance( + printIds, verbose, verbose, verbose, false, false, + processTime, handlerArgs); + } catch (NoSuchMethodException e) { + // Retry, this time matching the StdOutVisitorHandler constructor + // arg list + Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class, + boolean.class, boolean.class, boolean.class, + boolean.class, int.class, boolean.class }; + Constructor<?> cons = cls.getConstructor(consTypes); + handler = (VdsVisitHandler)cons.newInstance( + printIds, verbose, verbose, verbose, false, false, processTime, false); + } + + VisitorDataHandler dataHandler = handler.getDataHandler(); + VisitorControlHandler controlHandler = handler.getControlHandler(); + + VisitorDestinationParameters params = new VisitorDestinationParameters( + "visit-destination", dataHandler); + session = access.createVisitorDestinationSession(params); + while (!controlHandler.isDone()) { + Thread.sleep(1000); + } + } + + private void initShutdownHook() { + Runtime.getRuntime().addShutdownHook(new CleanUpThread()); + } + + class CleanUpThread extends Thread { + public void run() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException ise) { + //ignore this + } + try { + if (access != null) { + access.shutdown(); + } + } catch (IllegalStateException ise) { + //ignore this too + } + } + } +} diff --git a/vespaclient-java/src/main/sh/vds-document-statistics.sh b/vespaclient-java/src/main/sh/vds-document-statistics.sh new file mode 100755 index 00000000000..3677137dbd4 --- /dev/null +++ b/vespaclient-java/src/main/sh/vds-document-statistics.sh @@ -0,0 +1,20 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +function help { + echo "Usage: vds-document-statistics [ category, ... ]" + echo " Where category is one or more of: user, group, scheme, namespace" + echo "" + echo "vds-document-statistics generates documents counts based on one or more categories." + exit 0 +} +if [ "$1" == "-h" ]; then + help +fi +if [ "$1" == "" ]; then + help +fi +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java -Xms32m -Xmx128m $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.Main --statistics "$1" diff --git a/vespaclient-java/src/main/sh/vdsstat.sh b/vespaclient-java/src/main/sh/vdsstat.sh new file mode 100644 index 00000000000..ef3e2cdbe20 --- /dev/null +++ b/vespaclient-java/src/main/sh/vdsstat.sh @@ -0,0 +1,13 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespastat.Main "$@" diff --git a/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh new file mode 100755 index 00000000000..1a70fcb006d --- /dev/null +++ b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh @@ -0,0 +1,6 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +java $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.search.query.profile.DumpTool $@ diff --git a/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh new file mode 100755 index 00000000000..7534639d07b --- /dev/null +++ b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh @@ -0,0 +1,15 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export VESPA_LOG_TARGET=file:/dev/null +export MALLOC_ARENA_MAX=1 # Does not need fast allocation +java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespasummarybenchmark.VespaSummaryBenchmark "$@" diff --git a/vespaclient-java/src/main/sh/vespadestination.sh b/vespaclient-java/src/main/sh/vespadestination.sh new file mode 100755 index 00000000000..f2168cb6db2 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespadestination.sh @@ -0,0 +1,12 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.dummyreceiver.DummyReceiver "$@" diff --git a/vespaclient-java/src/main/sh/vespafeeder.sh b/vespaclient-java/src/main/sh/vespafeeder.sh new file mode 100755 index 00000000000..f74bc794ed7 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespafeeder.sh @@ -0,0 +1,15 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export VESPA_LOG_TARGET=file:/dev/null +export MALLOC_ARENA_MAX=1 # Does not need fast allocation +java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespafeeder.VespaFeeder "$@" diff --git a/vespaclient-java/src/main/sh/vespaget.sh b/vespaclient-java/src/main/sh/vespaget.sh new file mode 100644 index 00000000000..514ff170742 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespaget.sh @@ -0,0 +1,14 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.awt.headless=true \ +-DVESPA_LOG_LEVEL="all -debug -spam -config -info -event" \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespaget.Main "$@" diff --git a/vespaclient-java/src/main/sh/vespavisit.1 b/vespaclient-java/src/main/sh/vespavisit.1 new file mode 100644 index 00000000000..b9a6c488bf9 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisit.1 @@ -0,0 +1,159 @@ +.TH VESPAVISIT 1 2008-03-07 "Vespa" "Vespa Documentation" +.SH NAME +vespavisit \- Visit documents from a Vespa installation +.SH SYNPOSIS +.B vespavisit +[\fIOPTION\fR]... +.SH DESCRIPTION +.PP +In the regular case, retrieve documents stored in VESPA, and either print +them to STDOUT or send them to a given MessageBus route. +.PP +A Vespa visit operation processes a set of stored documents, in undefined +order, locally on the storage nodes where they are stored. A visitor library +available on all storage nodes will receive the documents stored locally, and +can process these and send messages to the visitor data handler. The regular +case is to use the DumpVisitor library to merely send the documents themselves +in blocks back to the data handler, which by default is this client that will +write the documents to STDOUT. +.PP +Mandatory arguments to long options are mandatory for short options too. +Short options can not currently be concatenated together. +.TP +\fB\-s\fR, \fB\-\-selection\fR \fISELECTION\fR +A document selection string, specifying what documents to visit. Documentation +on the language itself can be found in the documentation. Note that this argument +should probably be quoted to prevent your shell from invalidating your +selection. +.TP +\fB\-f\fR, \fB\-\-from\fR \fITIME\fR +If this option is given, only documents from given timestamp or newer will be +visited. The time is given in microseconds since 1970. +.TP +\fB\-t\fR, \fB\-\-to\fR \fITIME\fR +If this option is given, only documents up to and including the given timestamp +will be visited. The time is given in microseconds since 1970. +.TP +\fB\-e\fR, \fB\-\-headersonly\fR +By default, the whole documents stored are processed. If this option is given +only the header parts of documents will be processed. By defining the big +document fields as body fields, you can efficiently visit all the header fields +using this option. +.TP +\fB\-i\fR, \fB\-\-printids\fR +Using this option, only the document identifiers will be printed to STDOUT. +In addition, if visiting removes, an additional tag will be added so you can +see whether document has been removed or not. This option implies headers only +visiting, and can only be used if no datahandler is specified. +.TP +\fB\-d\fR, \fB\-\-datahandler\fR \fIVISITTARGET\fR +The data handler is the destination of messages sent from the visitor library. +By default, the data handler is the vespavisit process you start, which will +merely print all returned data to STDOUT. A visit target can be specified +instead. See the chapter below on visit targets. +.TP +\fB\-p\fR, \fB\-\-progress\fR \fIFILE\fR +By setting a progress file, current visitor progress will be saved to this +file at regular intervals. If this file exists on startup, the visitor will +continue from this point. +.TP +\fB\-o\fR, \fB\-\-timeout\fR \fITIMEOUT\fR +Time out the visitor after given number of milliseconds. +.TP +\fB\-r\fR, \fB\-\-visitremoves\fR +By default, only documents existing in Vespa will be processed. By giving +this option, also entries identifying documents previously existing will +be returned. This is useful for secondary copies of data that wants to know +whether documents it has stored has been removed. Note that documents deleted +a long time ago will no longer be tracked. Vespa keeps remove entries for +a configurable amount of time. +.TP +\fB\-m\fR, \fB\-\-maxpending\fR \fINUM\fR +Maximum pending docblock messages to data handlers. This may be used to +increase or reduce visiting speed, but should not be set too high so that data +handlers run out of memory. To get an estimate of memory consumption on each +data handler, multiply maxpending with defaultdocblocksize in stor-visitor +config and divide by number of data handlers. Default value for maxpending is +16. +.TP +\fB\-c\fR, \fB\-\-cluster\fR \fICLUSTER\fR +Visit the given VDS cluster. +.TP +\fB\-v\fR, \fB\-\-verbose\fR +More verbose output. Indent XML and add progress and info to STDERR. +.TP +\fB\-h\fR, \fB\-\-help\fR +Shows a short syntax reminder. +.PP +Advanced options: +.PP +The below options are used for advanced usage or for testing. +.TP +\fB\-\-visitlibrary\fR \fILIBRARY\fR +By default, the DumpVisitor library, sending documents back to the data handler, +is used when visiting. Another library can be specified using this option. The +library filename should be the name given here, with lib prepended and .so +appended. +.TP +\fB\-\-libraryparam\fR \fIKEY\fR \fIVALUE\fR +The default DumpVisitor library has no options to set, but custom libraries +may need user specifiable options. Here such options can be specified. Look +at visitor library documentation for legal parameters. +.TP +\fB\-\-polling\fR \fIarg\fR +The document API implements both a polling and a callback visitor API. The +callback API is most efficient and used by default. The polling API might be +simpler for users used to such APIs. Some VESPA system tests use this option +to test that the polling API works. +.TP +\fB\-\-visitinconsistentbuckets\fR +In some cases Vespa may temporarily be in an inconsistent state, that is, +different nodes contain different copies of the data. Collections of documents +are grouped into so-called buckets. The normal behavior of visiting is to wait +for the inconsistencies to resolve before actually visiting the data. This +might be a problem for time critical applications. Setting this option will +result in the bucket copy with most documents to be visited in case of +inconsistencies, which means that the data returned by the visitor are not +guaranteed to be correct. +.SH VISIT TARGET +Results from visiting can be sent to many different kind of targets. +.TP +\fBMessage bus routes\fR +You can specify a message bus route name directly, and this route will be used +to send the results. This is typically used when doing reprocessing within +Vespa. Message bus routes are set up in the application package. In addition +some routes may have been autogenerated in simple setups, for instance a +route called \fIdefault\fR is generated if your setup is so simple that Vespa +can guess where you want to send your data. +.TP +\fBSlobrok address\fR +You can also specify a slobrok address for data to be sent to. A slobrok address +is a slash separated path where you can use asterisk to mean any element within +this path. For instance, if you have a docproc cluster called \fImydpcluster\fR +it will have registered its nodes with slobrok names like +\fIdocproc/cluster.mydpcluster/docproc/0/feed_processor\fR, where the 0 here +indicates the first node in the cluster. You can thus specify to send visit data +to this docproc cluster by stating a slobrok address of +\fIdocproc/cluster.mydpcluster/docproc/*/feed_processor\fR. Note that this will +not send all the data to one or all the nodes. The data sent from the visitor +will be distributed among the matching nodes, but each message will just be sent +to one node. + +Slobrok names may also be used if you use the \fBvespavisittarget\fR tool to +retrieve the data at some location. If you start vespavisittarget on two nodes, +listening to slobrok names \fImynode/0/visit-destination\fR and +\fImynode/1/visit-destination\fR you can send the results to these nodes by +specifying \fImynode/*/visit-destination\fR as the data handler. See +\fBman vespavisittarget\fR for naming conventions used for such targets. +.TP +\fBTCP socket\fR +TCP sockets can also be specified directly. This requires that the endpoint +speaks FNET RPC though. This is typically done, either by using the +\fBvespavisittarget\fR tool, or by using a visitor destination programmatically +by using utility class in the document API. A socket address looks like the +following: tcp/\fIhostname\fR:\fIport\fR/\fIservicename\fR. For instance, an +address generated by the \fBvespavisittarget\fR tool might look like the +following: \fItcp/myhost.com:12345/visit-destination\fR. + +.SH AUTHOR +Written by Haakon Humberset. diff --git a/vespaclient-java/src/main/sh/vespavisit.sh b/vespaclient-java/src/main/sh/vespavisit.sh new file mode 100755 index 00000000000..eb6c9487b88 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisit.sh @@ -0,0 +1,14 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.VdsVisit "$@" diff --git a/vespaclient-java/src/main/sh/vespavisittarget.1 b/vespaclient-java/src/main/sh/vespavisittarget.1 new file mode 100644 index 00000000000..7f02215d558 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisittarget.1 @@ -0,0 +1,40 @@ +.TH VESPAVISITTARGET 1 2008-03-07 "Vespa" "Vespa Documentation" +.SH NAME +vespavisittarget \- An endpoint for documents visited from a Vespa installation +.SH SYNPOSIS +.B vespavisittarget +[\fIOPTION\fR]... +.SH DESCRIPTION +.PP +When visiting data from Vespa, you might not want to send the data back to the +controlling process. By using separate visitor targets you can divide load +between multiple nodes and have the controlling process run at another location. +The document API has utility classes to set up end points for visitor data from +Vespa. This application is a small tool that uses these utilities and merely +writes the data retrieved to STDOUT in XML format. +.PP +Mandatory arguments to long options are mandatory for short options too. +Short options can not currently be concatenated together. +.TP +\fB\-s\fR, \fB\-\-bindtoslobrok\fR \fISLOBROKADDRESS\fR +Bind to the given slobrok address. Note that the value \fI/visit-destination\fR +will be appended to the given address. +.TP +\fB\-t\fR, \fB\-\-bindtosocket\fR \fIPORT\fR +Bind to the given TCP socket. This will make sure we listen to the given port. +No slobrok registration is done using this option, so you need to specify +TCP socket address in visitors to get data sent to this destination. +.TP +\fB\-i\fR, \fB\-\-printids\fR +Using this option, only the document identifiers will be printed to STDOUT. +In addition, if visiting removes, an additional tag will be added so you can +see whether document has been removed or not. This option implies headers only +visiting, and can only be used if no datahandler is specified. +.TP +\fB\-v\fR, \fB\-\-verbose\fR +More verbose output. Indent XML. +.TP +\fB\-h\fR, \fB\-\-help\fR +Shows a short syntax reminder. +.SH AUTHOR +Written by Haakon Humberset. diff --git a/vespaclient-java/src/main/sh/vespavisittarget.sh b/vespaclient-java/src/main/sh/vespavisittarget.sh new file mode 100755 index 00000000000..7eb9fe17e04 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisittarget.sh @@ -0,0 +1,13 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.vespavisit.VdsVisitTarget "$@" diff --git a/vespaclient-java/src/test/files/documentmanager.cfg b/vespaclient-java/src/test/files/documentmanager.cfg new file mode 100644 index 00000000000..966361d1fe3 --- /dev/null +++ b/vespaclient-java/src/test/files/documentmanager.cfg @@ -0,0 +1,113 @@ +enablecompression false +datatype[10] +datatype[0].id 1002 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 2 +datatype[0].weightedsettype[0] +datatype[0].structtype[0] +datatype[0].documenttype[0] +datatype[1].id 1000 +datatype[1].arraytype[1] +datatype[1].arraytype[0].datatype 0 +datatype[1].weightedsettype[0] +datatype[1].structtype[0] +datatype[1].documenttype[0] +datatype[2].id 1004 +datatype[2].arraytype[1] +datatype[2].arraytype[0].datatype 4 +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[0] +datatype[3].id 1016 +datatype[3].arraytype[1] +datatype[3].arraytype[0].datatype 16 +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[4].id 1001 +datatype[4].arraytype[1] +datatype[4].arraytype[0].datatype 1 +datatype[4].weightedsettype[0] +datatype[4].structtype[0] +datatype[4].documenttype[0] +datatype[5].id 2001 +datatype[5].arraytype[0] +datatype[5].weightedsettype[1] +datatype[5].weightedsettype[0].datatype 0 +datatype[5].weightedsettype[0].createifnonexistant false +datatype[5].weightedsettype[0].removeifzero false +datatype[5].structtype[0] +datatype[5].documenttype[0] +datatype[6].id 2002 +datatype[6].arraytype[0] +datatype[6].weightedsettype[1] +datatype[6].weightedsettype[0].datatype 2 +datatype[6].weightedsettype[0].createifnonexistant false +datatype[6].weightedsettype[0].removeifzero false +datatype[6].structtype[0] +datatype[6].documenttype[0] +datatype[7].id -628990518 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name news.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[6] +datatype[7].structtype[0].field[0].name url +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype 10 +datatype[7].structtype[0].field[1].name title +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype 2 +datatype[7].structtype[0].field[2].name last_downloaded +datatype[7].structtype[0].field[2].id[0] +datatype[7].structtype[0].field[2].datatype 0 +datatype[7].structtype[0].field[3].name value_long +datatype[7].structtype[0].field[3].id[0] +datatype[7].structtype[0].field[3].datatype 4 +datatype[7].structtype[0].field[4].name value_content +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 2 +datatype[7].structtype[0].field[5].name stringarr +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 1002 +datatype[7].documenttype[0] +datatype[8].id 538588767 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name news.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[7] +datatype[8].structtype[0].field[0].name intarr +datatype[8].structtype[0].field[0].id[0] +datatype[8].structtype[0].field[0].datatype 1000 +datatype[8].structtype[0].field[1].name longarr +datatype[8].structtype[0].field[1].id[0] +datatype[8].structtype[0].field[1].datatype 1004 +datatype[8].structtype[0].field[2].name bytearr +datatype[8].structtype[0].field[2].id[0] +datatype[8].structtype[0].field[2].datatype 1016 +datatype[8].structtype[0].field[3].name floatarr +datatype[8].structtype[0].field[3].id[0] +datatype[8].structtype[0].field[3].datatype 1001 +datatype[8].structtype[0].field[4].name weightedsetint +datatype[8].structtype[0].field[4].id[0] +datatype[8].structtype[0].field[4].datatype 2001 +datatype[8].structtype[0].field[5].name weightedsetstring +datatype[8].structtype[0].field[5].id[0] +datatype[8].structtype[0].field[5].datatype 2002 +datatype[8].structtype[0].field[6].name content +datatype[8].structtype[0].field[6].id[0] +datatype[8].structtype[0].field[6].datatype 2 +datatype[8].documenttype[0] +datatype[9].id -1048827947 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name news +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -628990518 +datatype[9].documenttype[0].bodystruct 538588767 diff --git a/vespaclient-java/src/test/files/malformedfeed.json b/vespaclient-java/src/test/files/malformedfeed.json new file mode 100644 index 00000000000..26691ada676 --- /dev/null +++ b/vespaclient-java/src/test/files/malformedfeed.json @@ -0,0 +1,13 @@ +[ +{ + "put": "id:test:news::foo", + "fields": {} +}, +{ + "update": "id:test:news::foo", + "fields": {} +}, +{ + "remove": "id:test:news::foo" +}, +] diff --git a/vespaclient-java/src/test/files/myfeed.json b/vespaclient-java/src/test/files/myfeed.json new file mode 100644 index 00000000000..544f370e62a --- /dev/null +++ b/vespaclient-java/src/test/files/myfeed.json @@ -0,0 +1,13 @@ +[ +{ + "put": "id:test:news::foo", + "fields": {} +}, +{ + "update": "id:test:news::foo", + "fields": {} +}, +{ + "remove": "id:test:news::foo" +} +] diff --git a/vespaclient-java/src/test/files/myfeed.xml b/vespaclient-java/src/test/files/myfeed.xml new file mode 100644 index 00000000000..008d4fcba13 --- /dev/null +++ b/vespaclient-java/src/test/files/myfeed.xml @@ -0,0 +1,5 @@ +<vespafeed> + <document documentid="doc:test:foo" documenttype="news"/> + <update documentid="doc:test:foo" documenttype="news"/> + <remove documentid="doc:test:foo"/> +</vespafeed>
\ No newline at end of file diff --git a/vespaclient-java/src/test/files/progress.txt b/vespaclient-java/src/test/files/progress.txt new file mode 100644 index 00000000000..77323926140 --- /dev/null +++ b/vespaclient-java/src/test/files/progress.txt @@ -0,0 +1,8 @@ +VDS bucket progress file +14 +4704 +4701 +16384 +3800000000000004:8c00000600000004 +3800000000000002:8c00000000000002 +3800000000003e92:0 diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java new file mode 100644 index 00000000000..2b0b4cc9048 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java @@ -0,0 +1,77 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.EmptyReply; +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + */ +public class BenchmarkProgressPrinterTest extends TestCase { + + class DummyTimer implements Timer { + long ms; + + public long milliTime() { return ms; } + } + + public void testSimple() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + DummyTimer timer = new DummyTimer(); + timer.ms = 0; + BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output)); + RouteMetricSet metrics = new RouteMetricSet("foobar", printer); + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1)); + metrics.addReply(reply); + } + + timer.ms = 1200; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3)); + metrics.addReply(reply); + } + + timer.ms = 2400; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(4)); + reply.addError(new com.yahoo.messagebus.Error(32, "foo")); + metrics.addReply(reply); + } + + timer.ms = 62000; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5)); + reply.addError(new com.yahoo.messagebus.Error(64, "bar")); + metrics.addReply(reply); + } + + metrics.done(); + + String val = output.toString().split("\n")[1]; + + String correctPattern = "62000,\\s*3,\\s*2,\\s*\\d+,\\s*\\d+,\\s*\\d+$"; + assertTrue("Value '" + val + "' does not match pattern '" + correctPattern + "'", val.matches(correctPattern)); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java new file mode 100644 index 00000000000..ae49bb1318d --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java @@ -0,0 +1,90 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.EmptyReply; +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + */ +public class ProgressPrinterTest extends TestCase { + + class DummyTimer implements Timer { + long ms; + + public long milliTime() { return ms; } + } + + public void testSimple() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + DummyTimer timer = new DummyTimer(); + timer.ms = 0; + ProgressPrinter printer = new ProgressPrinter(timer, new PrintStream(output)); + RouteMetricSet metrics = new RouteMetricSet("foobar", printer); + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1)); + metrics.addReply(reply); + } + + timer.ms = 1200; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3)); + metrics.addReply(reply); + } + + timer.ms = 2400; + + { + DocumentIgnoredReply reply = new DocumentIgnoredReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(0)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5)); + reply.addError(new com.yahoo.messagebus.Error(32, "foo")); + metrics.addReply(reply); + } + + timer.ms = 62000; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(6)); + reply.addError(new com.yahoo.messagebus.Error(64, "bar")); + metrics.addReply(reply); + } + + String val = output.toString().replaceAll("latency\\(min, max, avg\\): .*", "latency(min, max, avg): 0, 0, 0"); + + String correct = + "\rSuccessfully sent 2 messages so far" + + "\rSuccessfully sent 3 messages so far" + + "\n\n" + + "Messages sent to vespa (route foobar) :\n" + + "---------------------------------------\n" + + "PutDocument:\tok: 2 msgs/sec: 0.03 failed: 0 ignored: 1 latency(min, max, avg): 0, 0, 0\n" + + "UpdateDocument:\tok: 1 msgs/sec: 0.02 failed: 2 ignored: 0 latency(min, max, avg): 0, 0, 0\n"; + + assertEquals(correct, val); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java new file mode 100644 index 00000000000..42d4b082ff3 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -0,0 +1,208 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import static org.junit.Assert.*; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedhandler.VespaFeedHandler; +import com.yahoo.text.Utf8; +import com.yahoo.vespaclient.config.FeederConfig; +import com.yahoo.vespafeeder.Arguments.HelpShownException; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class VespaFeederTestCase { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testParseArgs() throws Exception { + String argsS="--abortondataerror false --abortonsenderror false --file foo.xml --maxpending 10" + + " --maxpendingsize 11 --maxfeedrate 29 --mode benchmark --noretry --retrydelay 12 --route e6 --timeout 13 --trace 4" + + " --validate -v bar.xml --priority LOW_1"; + + Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); + + FeederConfig config = arguments.getFeederConfig(); + assertEquals(false, config.abortondocumenterror()); + assertEquals(13.0, config.timeout(), 0.00001); + assertEquals(false, config.retryenabled()); + assertEquals(12.0, config.retrydelay(), 0.0001); + assertEquals("e6", config.route()); + assertEquals(4, config.tracelevel()); + assertEquals(false, config.abortonsenderror()); + assertEquals(10, config.maxpendingdocs()); + assertEquals(11, config.maxpendingbytes()); + assertEquals(29.0, config.maxfeedrate(), 0.0001); + assertTrue(arguments.isVerbose()); + assertFalse(config.createifnonexistent()); + + assertEquals("LOW_1", arguments.getPriority()); + assertEquals("benchmark", arguments.getMode()); + assertEquals("foo.xml", arguments.getFiles().get(0)); + assertEquals("bar.xml", arguments.getFiles().get(1)); + } + + @Test + public void requireThatCreateIfNonExistentArgumentCanBeParsed() throws Exception { + String argsS="--create-if-non-existent --file foo.xml"; + Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); + assertTrue(arguments.getFeederConfig().createifnonexistent()); + } + + @Test + public void testHelp() throws Exception { + String argsS="-h"; + + try { + new Arguments(argsS.split(" "), null); + assertTrue(false); + } catch (Arguments.HelpShownException e) { + + } + } + + @Test + public void requireCorrectInputTypeDetection() throws IOException { + { + BufferedInputStream b = new BufferedInputStream( + new ByteArrayInputStream(Utf8.toBytes("[]"))); + InputStreamRequest r = new InputStreamRequest(b); + VespaFeeder.setJsonInput(r, b); + assertEquals("true", r.getProperty(VespaFeedHandler.JSON_INPUT)); + } + { + BufferedInputStream b = new BufferedInputStream( + new ByteArrayInputStream(Utf8.toBytes("<document />"))); + InputStreamRequest r = new InputStreamRequest(b); + VespaFeeder.setJsonInput(r, b); + assertEquals("false", r.getProperty(VespaFeedHandler.JSON_INPUT)); + } + } + + public void assertRenderErrorOutput(String expected, String[] errors) { + ArrayList<String> l = new ArrayList<String>(); + l.addAll(Arrays.asList(errors)); + assertEquals(expected, VespaFeeder.renderErrors(l).getMessage()); + } + + @Test + public void testRenderErrors() { + { + String[] errors = { "foo" }; + assertRenderErrorOutput("Errors:\n" + + "-------\n" + + " foo\n", errors); + } + + { + String[] errors = { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}; + assertRenderErrorOutput("First 10 errors (of 11):\n" + + "------------------------\n" + + " 1\n 2\n 3\n 4\n 5\n 6\n 7\n 8\n 9\n 10\n", errors); + } + } + + public RouteMetricSet.ProgressCallback getProgressPrinter(String args) throws Exception { + Arguments arguments = new Arguments(args.split(" "), DummySessionFactory.createWithAutoReply()); + return new VespaFeeder(arguments, null).createProgressCallback(System.out); + } + + @Test + public void testCreateProgressPrinter() throws Exception { + assert(getProgressPrinter("--mode benchmark") instanceof BenchmarkProgressPrinter); + assert(getProgressPrinter("") instanceof ProgressPrinter); + } + + private static class FeedFixture { + DummySessionFactory sessionFactory = DummySessionFactory.createWithAutoReply(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + DocumentTypeManager typeManager = new DocumentTypeManager(); + FeedFixture() { + DocumentTypeManagerConfigurer.configure(typeManager, "file:src/test/files/documentmanager.cfg"); + } + } + + @Test + public void feedFile() throws Exception { + FeedFixture f = new FeedFixture(); + Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --priority LOW_1".split(" "), f.sessionFactory); + new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream); + + assertEquals(3, f.sessionFactory.messages.size()); + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority()); + assertEquals("doc:test:foo", ((PutDocumentMessage) f.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); + DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertEquals("doc:test:foo", update.getId().toString()); + assertFalse(update.getCreateIfNonExistent()); + assertEquals("doc:test:foo", ((RemoveDocumentMessage) f.sessionFactory.messages.get(2)).getDocumentId().toString()); + + assertTrue(f.outputStream.toString().contains("Messages sent to vespa")); + } + + @Test + public void feedJson() throws Exception { + FeedFixture feedFixture = feed("src/test/files/myfeed.json", true); + + assertJsonFeedState(feedFixture); + } + + protected void assertJsonFeedState(FeedFixture feedFixture) { + assertEquals(3, feedFixture.sessionFactory.messages.size()); + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority()); + assertEquals("id:test:news::foo", ((PutDocumentMessage) feedFixture.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); + DocumentUpdate update = ((UpdateDocumentMessage) feedFixture.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertEquals("id:test:news::foo", update.getId().toString()); + assertFalse(update.getCreateIfNonExistent()); + assertEquals("id:test:news::foo", ((RemoveDocumentMessage) feedFixture.sessionFactory.messages.get(2)).getDocumentId().toString()); + + assertTrue(feedFixture.outputStream.toString().contains("Messages sent to vespa")); + } + + @Test + public void requireThatCreateIfNonExistentArgumentIsUsed() throws Exception { + FeedFixture f = new FeedFixture(); + Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --create-if-non-existent".split(" "), f.sessionFactory); + new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream); + + assertEquals(3, f.sessionFactory.messages.size()); + DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertTrue(update.getCreateIfNonExistent()); + } + + @Test + public void feedMalformedJson() throws Exception { + exception.expect(VespaFeeder.FeedErrorException.class); + exception.expectMessage("JsonParseException"); + feed("src/test/files/malformedfeed.json", false); + } + + protected FeedFixture feed(String feed, boolean abortOnDataError) throws HelpShownException, + FileNotFoundException, Exception { + String abortOnDataErrorArgument = abortOnDataError ? "" : " --abortondataerror no"; + FeedFixture feedFixture = new FeedFixture(); + Arguments arguments = new Arguments(("--file " + + feed + + " --priority LOW_1" + abortOnDataErrorArgument).split(" "), feedFixture.sessionFactory); + new VespaFeeder(arguments, feedFixture.typeManager).parseFiles(System.in, feedFixture.printStream); + return feedFixture; + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java new file mode 100644 index 00000000000..3e707b04256 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java @@ -0,0 +1,195 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.*; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Test class for {@link CommandLineOptions} + * + * @author bjorncs + * @since 5.26 + */ +public class CommandLineOptionsTest { + + private final InputStream emptyStream = new InputStream() { + + @Override + public int read() throws IOException { + return -1; + } + }; + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private ClientParameters getParsedOptions(InputStream in, String... args) { + CommandLineOptions options = new CommandLineOptions(in); + return options.parseCommandLineArguments(args); + } + + private ClientParameters getParsedOptions(String... args) { + return getParsedOptions(emptyStream, args); + } + + @Test + public void testDefaultOptions() { + ClientParameters params = getParsedOptions(); + assertFalse(params.help); + assertFalse(params.documentIds.hasNext()); + assertFalse(params.printIdsOnly); + assertEquals("[all]", params.fieldSet); + assertEquals("default", params.route); + assertTrue(params.cluster.isEmpty()); + assertEquals("client", params.configId); + assertFalse(params.showDocSize); + assertEquals(0, params.timeout, 0); + assertFalse(params.noRetry); + assertEquals(0, params.traceLevel); + assertEquals(DocumentProtocol.Priority.NORMAL_2, params.priority); + assertTrue(params.loadTypeName.isEmpty()); + } + + @Test + public void testValidOptions() { + ClientParameters params = getParsedOptions( + "--fieldset", "[fieldset]", + "--route", "dummyroute", + "--configid", "dummyconfig", + "--showdocsize", + "--timeout", "0.25", + "--noretry", + "--trace", "1", + "--priority", Integer.toString(DocumentProtocol.Priority.HIGH_3.getValue()), + "--loadtype", "dummyloadtype", + "id:1", "id:2" + ); + + assertEquals("[fieldset]", params.fieldSet); + assertEquals("dummyroute", params.route); + assertEquals("dummyconfig", params.configId); + assertTrue(params.showDocSize); + assertEquals(0.25, params.timeout, 0.0001); + assertTrue(params.noRetry); + assertEquals(1, params.traceLevel); + assertEquals(DocumentProtocol.Priority.HIGH_3, params.priority); + assertEquals("dummyloadtype", params.loadTypeName); + + Iterator<String> documentsIds = params.documentIds; + assertEquals("id:1", documentsIds.next()); + assertEquals("id:2", documentsIds.next()); + assertFalse(documentsIds.hasNext()); + } + + @Test + public void testInvalidCombination1() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Print ids and headers only options are mutually exclusive."); + getParsedOptions("--headersonly", "--printids"); + } + + @Test + public void testInvalidCombination2() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); + getParsedOptions("--headersonly", "--fieldset", "[header]"); + } + + @Test + public void testInvalidCombination3() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); + getParsedOptions("--printids", "--fieldset", "[header]"); + } + + @Test + public void testInvalidCombination4() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Cluster and route options are mutually exclusive."); + getParsedOptions("--route", "dummyroute", "--cluster", "dummycluster"); + } + + @Test + public void testInvalidPriority() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid priority: 16"); + getParsedOptions("--priority", "16"); + } + + @Test + public void testInvalidTraceLevel1() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid tracelevel: -1"); + getParsedOptions("--trace", "-1"); + } + + @Test + public void testInvalidTraceLevel2() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid tracelevel: 10"); + getParsedOptions("--trace", "10"); + } + + @Test + public void testPrintids() { + ClientParameters params = getParsedOptions("--printids"); + assertEquals("[id]", params.fieldSet); + } + + @Test + public void testHeadersOnly() { + ClientParameters params = getParsedOptions("--headersonly"); + assertEquals("[header]", params.fieldSet); + } + + @Test + public void testCluster() { + ClientParameters params = getParsedOptions("--cluster", "dummycluster"); + assertEquals("dummycluster", params.cluster); + assertTrue(params.route.isEmpty()); + } + + @Test + public void testHelp() { + ClientParameters params = getParsedOptions("--help"); + assertTrue(params.help); + } + + @Test + public void testDocumentIdsFromInputStream() throws UnsupportedEncodingException { + InputStream in = new ByteArrayInputStream("id:1 id:2 id:3".getBytes("UTF-8")); + ClientParameters params = getParsedOptions(in, ""); + + Iterator<String> documentsIds = params.documentIds; + assertEquals("id:1", documentsIds.next()); + assertEquals("id:2", documentsIds.next()); + assertEquals("id:3", documentsIds.next()); + assertFalse(documentsIds.hasNext()); + } + + @Test + public void testPrintHelp() { + ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + PrintStream oldOut = System.out; + System.setOut(new PrintStream(outContent)); + try { + CommandLineOptions options = new CommandLineOptions(emptyStream); + options.printHelp(); + + String output = outContent.toString(); + assertTrue(output.contains("vespaget <options> [documentid...]")); + assertTrue(output.contains("Fetch a document from a Vespa Content cluster.")); + } finally { + System.setOut(oldOut); + outContent.reset(); + + } + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java new file mode 100644 index 00000000000..c3d3fcc71e9 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -0,0 +1,376 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Reply; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentMatcher; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link DocumentRetriever} + * + * @author bjorncs + */ +public class DocumentRetrieverTest { + + public static final String DOC_ID_1 = "id:storage_test:document::1"; + public static final String DOC_ID_2 = "id:storage_test:document::2"; + public static final String DOC_ID_3 = "id:storage_test:document::3"; + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + + private DocumentAccessFactory mockedFactory; + private MessageBusDocumentAccess mockedDocumentAccess; + private MessageBusSyncSession mockedSession; + private PrintStream oldOut; + private PrintStream oldErr; + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Before + public void setUpStreams() { + oldOut = System.out; + oldErr = System.err; + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @Before + public void prepareMessageBusMocks() { + this.mockedFactory = mock(DocumentAccessFactory.class); + this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class); + this.mockedSession = mock(MessageBusSyncSession.class); + when(mockedFactory.createDocumentAccess(any())).thenReturn(mockedDocumentAccess); + when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession); + } + + @After + public void cleanUpStreams() { + System.setOut(oldOut); + System.setErr(oldErr); + outContent.reset(); + errContent.reset(); + } + + private static ClientParameters.Builder createParameters() { + return new ClientParameters.Builder() + .setPriority(DocumentProtocol.Priority.NORMAL_2) + .setCluster("") + .setRoute("default") + .setConfigId("client") + .setFieldSet("[all]") + .setPrintIdsOnly(false) + .setHelp(false) + .setShowDocSize(false) + .setLoadTypeName("") + .setNoRetry(false) + .setTraceLevel(0) + .setTimeout(0) + .setDocumentIds(Collections.emptyIterator()); + } + + private static Iterator<String> asIterator(String... docIds) { + return Arrays.asList(docIds).iterator(); + } + + private static Reply createDocumentReply(String docId) { + return new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(docId))); + } + + private void assertContainsDocument(String documentId) { + assertTrue(outContent.toString().contains(String.format( + "<document documenttype=\"document\" documentid=\"%s\"/>", documentId))); + } + + private DocumentRetriever createDocumentRetriever(ClientParameters params) { + return createDocumentRetriever(params, new ClusterList()); + } + + private DocumentRetriever createDocumentRetriever(ClientParameters params, ClusterList clusterList) { + return new DocumentRetriever( + clusterList, + mockedFactory, + new LoadTypeSet(), + params); + } + + @Test + public void testSendSingleMessage() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPriority(DocumentProtocol.Priority.HIGH_1) + .setNoRetry(true) + .setLoadTypeName("loadtype") + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + LoadTypeSet loadTypeSet = new LoadTypeSet(); + loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1); + DocumentRetriever documentRetriever = new DocumentRetriever( + new ClusterList(), + mockedFactory, + loadTypeSet, + params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).syncSend(argThat(new ArgumentMatcher<GetDocumentMessage>() { + @Override + public boolean matches(Object o) { + GetDocumentMessage msg = (GetDocumentMessage) o; + return msg.getPriority().equals(DocumentProtocol.Priority.HIGH_1) && + !msg.getRetryEnabled() && + msg.getLoadType().equals(new LoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1)); + } + })); + assertContainsDocument(DOC_ID_1); + } + + @Test + public void testMultipleMessages() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3)) + .build(); + + when(mockedSession.syncSend(any())).thenReturn( + createDocumentReply(DOC_ID_1), + createDocumentReply(DOC_ID_2), + createDocumentReply(DOC_ID_3)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(3)).syncSend(any()); + assertContainsDocument(DOC_ID_1); + assertContainsDocument(DOC_ID_2); + assertContainsDocument(DOC_ID_3); + } + + @Test + public void testJsonOutput() throws DocumentRetrieverException, JsonParseException, IOException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3)) + .setJsonOutput(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn( + createDocumentReply(DOC_ID_1), + createDocumentReply(DOC_ID_2), + createDocumentReply(DOC_ID_3)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(3)).syncSend(any()); + ObjectMapper m = new ObjectMapper(); + @SuppressWarnings("unchecked") + List<Map<String, Object>> feed = m.readValue(outContent.toByteArray(), List.class); + assertEquals(DOC_ID_1, feed.get(0).get("id")); + assertEquals(DOC_ID_2, feed.get(1).get("id")); + assertEquals(DOC_ID_3, feed.get(2).get("id")); + } + + @Test + public void testShutdownHook() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + documentRetriever.shutdown(); + + verify(mockedSession, times(1)).destroy(); + verify(mockedDocumentAccess, times(1)).shutdown(); + } + + @Test + public void testInvalidLoadType() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("Loadtype with name 'undefinedloadtype' does not exist.\n"); + + ClientParameters params = createParameters() + .setLoadTypeName("undefinedloadtype") + .build(); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testClusterLookup() throws DocumentRetrieverException { + final String cluster = "storage", configId = "content/cluster.foo/storage", + expectedRoute = "[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]"; + + ClientParameters params = createParameters() + .setCluster(cluster) + .build(); + + ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef(cluster, configId))); + + DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); + documentRetriever.retrieveDocuments(); + + verify(mockedFactory).createDocumentAccess(argThat(new ArgumentMatcher<MessageBusParams>() { + @Override + public boolean matches(Object o) { + return ((MessageBusParams) o).getRoute().equals(expectedRoute); + } + })); + } + + @Test + public void testInvalidClusterName() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("The Vespa cluster contains the content clusters storage, not invalidclustername. Please select a valid vespa cluster."); + + ClientParameters params = createParameters() + .setCluster("invalidclustername") + .build(); + + ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef("storage", "content/cluster.foo/storage"))); + + DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testEmtpyClusterList() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("The Vespa cluster does not have any content clusters declared."); + + ClientParameters params = createParameters() + .setCluster("invalidclustername") + .build(); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testHandlingErrorFromMessageBus() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .build(); + + Reply r = new GetDocumentReply(null); + r.addError(new Error(0, "Error message")); + when(mockedSession.syncSend(any())).thenReturn(r); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertTrue(errContent.toString().contains("Request failed")); + } + + @Test + public void testShowDocSize() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setShowDocSize(true) + .build(); + + Document document = new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1)); + when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(document)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertTrue(outContent.toString().contains(String.format("Document size: %d bytes", document.getSerializedSize()))); + } + + @Test + public void testPrintIdOnly() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPrintIdsOnly(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertEquals(DOC_ID_1 + "\n", outContent.toString()); + } + + @Test + public void testDocumentNotFound() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPrintIdsOnly(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(null)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(outContent.toString(), "Document not found.\n"); + } + + @Test + public void testTrace() throws DocumentRetrieverException { + final int traceLevel = 9; + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setTraceLevel(traceLevel) + .build(); + + GetDocumentReply reply = new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1))); + reply.getTrace().getRoot().addChild("childnode"); + when(mockedSession.syncSend(any())).thenReturn(reply); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).setTraceLevel(traceLevel); + assertTrue(outContent.toString().contains("<trace>")); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java new file mode 100644 index 00000000000..2df8b2c4751 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java @@ -0,0 +1,87 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BucketStatsPrinterTest { + + private BucketStatsRetriever retriever; + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + @Before + public void mockBucketStatsRetriever() throws BucketStatsException { + retriever = mock(BucketStatsRetriever.class); + when(retriever.getBucketIdForType(any(), any())).thenReturn(new BucketId(0x42)); + when(retriever.retrieveBucketList(any())).thenReturn(Collections.emptyList()); + when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(""); + } + + @After + public void resetOutputMock() { + out.reset(); + } + + private String getOutputString() { + String content = out.toString(); + out.reset(); + return content; + } + + private String retreiveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException { + BucketStatsPrinter printer = new BucketStatsPrinter(retriever, new PrintStream(out)); + printer.retrieveAndPrintBucketStats(type, id, dumpData); + return getOutputString(); + } + + @Test + public void testShouldPrintBucketIdForUserAndGroup() throws BucketStatsException { + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains("Generated 32-bit bucket id")); + + output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.GROUP, "mygroup", false); + assertTrue(output.contains("Generated 32-bit bucket id")); + } + + @Test + public void testShouldPrintWarningIfBucketListEmpty() throws BucketStatsException { + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains("No actual files were stored for this bucket")); + } + + @Test + public void testShouldPrintBucketList() throws BucketStatsException { + List<GetBucketListReply.BucketInfo> bucketList = new ArrayList<>(); + String dummyInfoString = "dummyinformation"; + bucketList.add(new GetBucketListReply.BucketInfo(new BucketId(0), dummyInfoString)); + when(retriever.retrieveBucketList(any())).thenReturn(bucketList); + + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains(dummyInfoString)); + } + + @Test + public void testShouldPrintBucketStats() throws BucketStatsException { + String dummyBucketStats = "dummystats"; + GetBucketListReply.BucketInfo bucketInfo = new GetBucketListReply.BucketInfo(new BucketId(0), "dummy"); + when(retriever.retrieveBucketList(any())).thenReturn(Collections.singletonList(bucketInfo)); + when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(dummyBucketStats); + + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", true); + assertTrue(output.contains(dummyBucketStats)); + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java new file mode 100644 index 00000000000..38a79aa7b5d --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java @@ -0,0 +1,141 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.routing.Route; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BucketStatsRetrieverTest { + private final BucketIdFactory bucketIdFactory = new BucketIdFactory(); + + private DocumentAccessFactory mockedFactory; + private MessageBusDocumentAccess mockedDocumentAccess; + private MessageBusSyncSession mockedSession; + + + @Before + public void prepareMessageBusMocks() { + this.mockedFactory = mock(DocumentAccessFactory.class); + this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class); + this.mockedSession = mock(MessageBusSyncSession.class); + when(mockedFactory.createDocumentAccess()).thenReturn(mockedDocumentAccess); + when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession); + } + + @Test + public void testGetBucketId() throws BucketStatsException { + BucketStatsRetriever retriever = createRetriever(); + + assertEquals("BucketId(0x80000000000004d2)", + retriever.getBucketIdForType(ClientParameters.SelectionType.USER, "1234").toString()); + assertEquals("BucketId(0x800000003a7455d7)", + retriever.getBucketIdForType(ClientParameters.SelectionType.GROUP, "mygroup").toString()); + assertEquals("BucketId(0x800000003a7455d7)", + retriever.getBucketIdForType(ClientParameters.SelectionType.BUCKET, "0x800000003a7455d7").toString()); + assertEquals("BucketId(0xeb018ac5e5732db3)", + retriever.getBucketIdForType(ClientParameters.SelectionType.DOCUMENT, "id:ns:type::another").toString()); + assertEquals("BucketId(0xeadd5fe811a2012c)", + retriever.getBucketIdForType(ClientParameters.SelectionType.GID, "0x2c01a21163cb7d0ce85fddd6").toString()); + } + + @Test + public void testRetrieveBucketList() throws BucketStatsException { + String bucketInfo = "I like turtles!"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another")); + + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, bucketInfo)); + when(mockedSession.syncSend(any())).thenReturn(reply); + + List<GetBucketListReply.BucketInfo> bucketList = createRetriever().retrieveBucketList(bucketId); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(1, bucketList.size()); + assertEquals(bucketInfo, bucketList.get(0).getBucketInformation()); + } + + @Test + public void testRetrieveBucketStats() throws BucketStatsException { + String docId = "id:ns:type::another"; + String bucketInfo = "I like turtles!"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId(docId)); + + StatBucketReply reply = new StatBucketReply(); + reply.setResults(bucketInfo); + when(mockedSession.syncSend(any())).thenReturn(reply); + String result = createRetriever().retrieveBucketStats(ClientParameters.SelectionType.DOCUMENT, docId, bucketId); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(bucketInfo, result); + } + + @Test + public void testShutdownHook() { + class MockShutdownRegistrar implements BucketStatsRetriever.ShutdownHookRegistrar { + public Runnable shutdownRunnable; + @Override + public void registerShutdownHook(Runnable runnable) { + shutdownRunnable = runnable; + } + } + MockShutdownRegistrar registrar = new MockShutdownRegistrar(); + new BucketStatsRetriever(mockedFactory, "default", registrar); + registrar.shutdownRunnable.run(); + + verify(mockedSession, times(1)).destroy(); + verify(mockedDocumentAccess, times(1)).shutdown(); + } + + @Test(expected = BucketStatsException.class) + public void testShouldFailOnReplyError() throws BucketStatsException { + GetBucketListReply reply = new GetBucketListReply(); + reply.addError(new Error(0, "errormsg")); + when(mockedSession.syncSend(any())).thenReturn(reply); + + createRetriever().retrieveBucketList(new BucketId(1)); + } + + @Test + public void testRoute() throws BucketStatsException { + String route = "default"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another")); + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, "I like turtles!")); + when(mockedSession.syncSend(any())).thenReturn(reply); + + BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {}); + retriever.retrieveBucketList(new BucketId(0)); + + verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() { + @Override + public boolean matches(Object o) { + return ((Message) o).getRoute().equals(Route.parse(route)); + } + })); + } + + private BucketStatsRetriever createRetriever() { + return new BucketStatsRetriever(mockedFactory, "default", t -> {}); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java new file mode 100644 index 00000000000..e90c47e3150 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.*; + +public class CommandLineOptionsTest { + + private ClientParameters getParsedOptions(String... args) { + CommandLineOptions parser = new CommandLineOptions(); + return parser.parseCommandLineArguments(args); + } + + @Test + public void testHelp() { + assertTrue(getParsedOptions("--help").help); + } + + @Test + public void testMultipleOptions() { + ClientParameters params = getParsedOptions("--dump", "--route", "dummyroute", "--user", "userid"); + assertTrue(params.dumpData); + assertEquals("dummyroute", params.route); + assertEquals(ClientParameters.SelectionType.USER, params.selectionType); + assertEquals("userid", params.id); + } + + @Test + public void testSelectionTypes() { + assertEquals(ClientParameters.SelectionType.USER, getParsedOptions("--user", "id").selectionType); + assertEquals(ClientParameters.SelectionType.DOCUMENT, getParsedOptions("--document", "id").selectionType); + assertEquals(ClientParameters.SelectionType.BUCKET, getParsedOptions("--bucket", "id").selectionType); + assertEquals(ClientParameters.SelectionType.GROUP, getParsedOptions("--group", "id").selectionType); + assertEquals(ClientParameters.SelectionType.GID, getParsedOptions("--gid", "id").selectionType); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingSelectionType() { + getParsedOptions(); + } + + @Test(expected = IllegalArgumentException.class) + public void testFailOnMultipleDumpTypes() { + getParsedOptions("--user", "id", "--document", "id", "--group", "id", "--gid", "id"); + } + + @Test + public void testForceDumpOnDocumentOrGid() { + assertTrue(getParsedOptions("--document", "docid").dumpData); + assertTrue(getParsedOptions("--gid", "gid").dumpData); + } + + @Test + public void testPrintHelp() { + ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + PrintStream oldOut = System.out; + System.setOut(new PrintStream(outContent)); + try { + CommandLineOptions options = new CommandLineOptions(); + options.printHelp(); + String output = outContent.toString(); + assertTrue(output.contains("vdsstat [options]")); + } finally { + System.setOut(oldOut); + outContent.reset(); + } + } + + @Test + public void testDefaultRoute() { + assertEquals("default", getParsedOptions("--user", "dummyuser").route); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java new file mode 100644 index 00000000000..4eec05f7bc7 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class VdsVisitTargetTestCase { + + @Test + public void testParametersSlobrok() throws Exception { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments(new String[]{ + "--bindtoslobrok", "myname", + "--processtime", "34", + "--visithandler", "Foo", + "--visitoptions", "foo bar zoo", + "-i", + "-v" + }); + + assertEquals("myname", target.getSlobrokAddress()); + assertEquals(34, target.getProcessTime()); + assertEquals("Foo", target.getHandlerClassName()); + assertEquals(3, target.getHandlerArgs().length); + assertEquals("foo", target.getHandlerArgs()[0]); + assertEquals("bar", target.getHandlerArgs()[1]); + assertEquals("zoo", target.getHandlerArgs()[2]); + assertTrue(target.isVerbose()); + assertTrue(target.isPrintIds()); + } + + @Test + public void testParametersPort() throws Exception { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments("--bindtosocket 1234".split(" ")); + assertEquals(1234, target.getPort()); + assertEquals(null, target.getSlobrokAddress()); + } + + public void assertException(String params) { + try { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments(params.split(" ")); + assertTrue(false); + } catch (Exception e) { + + } + } + + @Test + public void testPortAndSlobrok() { + assertException("--bindtoslobrok foo --bindtosocket 1234"); + assertException("--bindtoport foo"); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java new file mode 100644 index 00000000000..49060a5715f --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java @@ -0,0 +1,475 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.messagebus.Trace; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import org.apache.commons.cli.Options; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +public class VdsVisitTestCase { + + private VdsVisit.ArgumentParser createMockArgumentParser() { + Options opts = VdsVisit.createOptions(); + return new VdsVisit.ArgumentParser(opts); + } + + @Test + public void testCommandLineShortOptions() throws Exception { + // short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-d", "foo.remote", + "-s", "'id.user=1234'", + "-f", "5678", + "-t", "9012", + "-l", "foodoc.bar,foodoc.baz", + "-m", "6000", + "-b", "5", + "-p", "foo-progress.txt", + "-u", "123456789", + "-c", "kittens", + "-r", + "-v" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("foo.remote", params.getRemoteDataHandler()); + assertEquals("'id.user=1234'", params.getDocumentSelection()); + assertEquals(5678, params.getFromTimestamp()); + assertEquals(9012, params.getToTimestamp()); + assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet()); + assertEquals(6000, params.getMaxPending()); + assertEquals(5, params.getMaxBucketsPerVisitor()); + assertEquals("foo-progress.txt", params.getResumeFileName()); + assertEquals(123456789, params.getTimeoutMs()); + assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout()); + assertEquals("kittens", allParams.getCluster()); + assertTrue(allParams.isVerbose()); + } + + /** + * Test the parameters that could not be used in conjunction with + * those in the first parameter test. + * @throws Exception + */ + @Test + public void testCommandLineShortOptions2() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-o", "654321", + "-e" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals(654321, allParams.getFullTimeout()); + assertEquals(654321, params.getTimeoutMs()); + assertEquals("[header]", params.getFieldSet()); + } + + @Test + public void testCommandLineShortOptionsPrintIdsOnly() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-i" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("[id]", params.getFieldSet()); + assertTrue(allParams.isPrintIdsOnly()); + } + + @Test + public void testCommandLineLongOptions() throws Exception { + // short options testing (for options that do not collide with each other) + String[] args = new String[] { + "--datahandler", "foo.remote", + "--selection", "'id.user=1234'", + "--from", "5678", + "--to", "9012", + "--fieldset", "foodoc.bar,foodoc.baz", + "--maxpending", "6000", + "--maxbuckets", "5", + "--progress", "foo-progress.txt", + "--maxpendingsuperbuckets", "3", + "--buckettimeout", "123456789", + "--cluster", "kittens", + "--visitinconsistentbuckets", + "--visitlibrary", "fnord", + "--libraryparam", "asdf", "rargh", + "--libraryparam", "pinkie", "pie", + "--processtime", "555", + "--maxhits", "1001", + "--maxtotalhits", "2002", + "--tracelevel", "8", + "--priority", "NORMAL_1", + "--ordering", "ascending", + "--skipbucketsonfatalerrors", + "--abortonclusterdown", + "--visitremoves" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + + assertEquals("foo.remote", params.getRemoteDataHandler()); + assertEquals("'id.user=1234'", params.getDocumentSelection()); + assertEquals(5678, params.getFromTimestamp()); + assertEquals(9012, params.getToTimestamp()); + assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet()); + assertEquals(6000, params.getMaxPending()); + assertEquals(5, params.getMaxBucketsPerVisitor()); + assertEquals("foo-progress.txt", params.getResumeFileName()); + assertEquals(123456789, params.getTimeoutMs()); + assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout()); + assertEquals("kittens", allParams.getCluster()); + + assertTrue(params.getThrottlePolicy() instanceof StaticThrottlePolicy); + assertEquals(3, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + + assertTrue(params.visitInconsistentBuckets()); + assertEquals("fnord", params.getVisitorLibrary()); + // TODO: FIXME? multiple library params doesn't work + assertTrue(Arrays.equals("rargh".getBytes(), params.getLibraryParameters().get("asdf"))); + //assertTrue(Arrays.equals("pie".getBytes(), params.getLibraryParameters().get("pinkie"))); + assertEquals(555, allParams.getProcessTime()); + assertEquals(1001, params.getMaxFirstPassHits()); + assertEquals(2002, params.getMaxTotalHits()); + assertEquals(8, params.getTraceLevel()); + assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); + assertEquals(OrderingSpecification.ASCENDING, params.getVisitorOrdering()); + assertTrue(allParams.getAbortOnClusterDown()); + assertTrue(params.visitRemoves()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + VdsVisit.verbosePrintParameters(allParams, printStream); + printStream.flush(); + String nl = System.getProperty("line.separator"); // the joys of running tests on windows + assertEquals( + "Time out visitor after 123456789 ms." + nl + + "Visiting documents matching: 'id.user=1234'" + nl + + "Visiting in the inclusive timestamp range 5678 - 9012." + nl + + "Visiting field set foodoc.bar,foodoc.baz." + nl + + "Visiting inconsistent buckets." + nl + + "Including remove entries." + nl + + "Tracking progress in file: foo-progress.txt" + nl + + "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor." + nl + + "Visit maximum 5 buckets per visitor." + nl + + "Sending data to data handler at: foo.remote" + nl + + "Using visitor library 'fnord'." + nl + + "Adding the following library specific parameters:" + nl + + " asdf = rargh" + nl + + "Visitor priority NORMAL_1" + nl + + "Skip visiting super buckets with fatal errors." + nl, + outputStream.toString("utf-8")); + + args = new String[] { + "--ordering", "descending" + }; + allParams = parser.parse(args); + params = allParams.getVisitorParameters(); + assertEquals(OrderingSpecification.DESCENDING, params.getVisitorOrdering()); + } + + private static String[] emptyArgList() { return new String[]{}; } + + @Test + public void visitor_priority_is_low1_by_default() throws Exception { + VdsVisit.VdsVisitParameters allParams = createMockArgumentParser().parse(emptyArgList()); + + VisitorParameters params = allParams.getVisitorParameters(); + assertEquals(DocumentProtocol.Priority.LOW_1, params.getPriority()); + } + + @Test + public void testBadPriorityValue() throws Exception { + String[] args = new String[] { + "--priority", "super_hyper_important" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + try { + parser.parse(args); + fail("no exception thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unknown priority name")); + } + } + + @Test + public void testBadOrderingValue() throws Exception { + String[] args = new String[] { + "--ordering", "yonder" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + try { + parser.parse(args); + fail("no exception thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unknown ordering")); + } + } + + @Test + public void testCommandLineShortOptionsInvokeHelp() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-h" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNull(allParams); + } + + @Test + public void testAutoSelectClusterRoute() throws Exception { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + + String route = VdsVisit.resolveClusterRoute(clusterList, null); + assertEquals("[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]", route); + } + + @Test + public void testBadClusterName() throws Exception { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, "borkbork"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Your vespa cluster contains the content clusters storage, not borkbork.")); + } + } + + @Test + public void testRequireClusterOptionIfMultipleClusters() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, null); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Please use the -c option to select one of them")); + } + } + + @Test + public void testExplicitClusterOptionWithMultipleClusters() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + + String route = VdsVisit.resolveClusterRoute(clusterList, "storage2"); + assertEquals("[Storage:cluster=storage2;clusterconfigid=content/cluster.bar/storage]", route); + } + + @Test + public void testFailIfNoContentClustersAvailable() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, null); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Your Vespa cluster does not have any content clusters")); + } + } + + @Test + public void testStatistics() throws Exception { + String[] args = new String[] { + "--statistics", "foo" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("foo", allParams.getStatisticsParts()); + assertEquals("[id]", params.getFieldSet()); + assertEquals("CountVisitor", params.getVisitorLibrary()); + } + + // TODO: use DummyVisitorSession instead? + private static class MockVisitorSession implements VisitorSession { + private VisitorParameters params; + + public MockVisitorSession(VisitorParameters params) { + this.params = params; + params.getLocalDataHandler().setSession(this); + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public ProgressToken getProgress() { + return null; + } + + @Override + public Trace getTrace() { + return null; + } + + @Override + public boolean waitUntilDone(long l) throws InterruptedException { + params.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "woo!"); + // Return immediately + return true; + } + + @Override + public void ack(AckToken ackToken) { + } + + @Override + public void abort() { + } + + @Override + public VisitorResponse getNext() { + return null; + } + + @Override + public VisitorResponse getNext(int i) throws InterruptedException { + return null; + } + + @Override + public void destroy() { + } + } + + private static class MockVisitorSessionAccessor implements VdsVisit.VisitorSessionAccessor { + boolean shutdown = false; + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return new MockVisitorSession(params); + } + + @Override + public void shutdown() { + shutdown = true; + } + + public boolean isShutdown() { + return shutdown; + } + } + + private static class MockVisitorSessionAccessorFactory implements VdsVisit.VisitorSessionAccessorFactory { + + private MockVisitorSessionAccessor lastCreatedAccessor = null; + + @Override + public VdsVisit.VisitorSessionAccessor createVisitorSessionAccessor() { + lastCreatedAccessor = new MockVisitorSessionAccessor(); + return lastCreatedAccessor; + } + + public MockVisitorSessionAccessor getLastCreatedAccessor() { + return lastCreatedAccessor; + } + } + + private static class MockShutdownHookRegistrar implements VdsVisit.ShutdownHookRegistrar { + Thread cleanUpThread; + + @Override + public void registerShutdownHook(Thread thread) { + cleanUpThread = thread; + } + + public Thread getCleanUpThread() { + return cleanUpThread; + } + } + + @Test + public void testVdsVisitRunLogic() { + MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory(); + MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar(); + VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar); + + VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters(); + VisitorParameters visitorParameters = new VisitorParameters(""); + params.setVisitorParameters(visitorParameters); + + visitorParameters.setResumeFileName("src/test/files/progress.txt"); + vdsVisit.setVdsVisitParameters(params); + + int code = vdsVisit.doRun(); + assertEquals(0, code); + + assertNotNull(shutdownHookRegistrar.getCleanUpThread()); + shutdownHookRegistrar.getCleanUpThread().run(); + + assertNotNull(accessorFactory.getLastCreatedAccessor()); + assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown()); + + // Ensure progress token stuff was read from file + ProgressToken progress = visitorParameters.getResumeToken(); + assertNotNull(progress); + assertEquals(14, progress.getDistributionBitCount()); + assertEquals(3, progress.getPendingBucketCount()); + } + + @Test + public void testVdsVisitRunLogicProgressFileNotYetCreated() { + MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory(); + MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar(); + VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar); + + VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters(); + VisitorParameters visitorParameters = new VisitorParameters(""); + params.setVisitorParameters(visitorParameters); + + visitorParameters.setResumeFileName("src/test/files/progress-not-existing.txt"); + vdsVisit.setVdsVisitParameters(params); + + // Should not fail with file not found + int code = vdsVisit.doRun(); + assertEquals(0, code); + + assertNotNull(shutdownHookRegistrar.getCleanUpThread()); + shutdownHookRegistrar.getCleanUpThread().run(); + + assertNotNull(accessorFactory.getLastCreatedAccessor()); + assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown()); + } +} diff --git a/vespaclient/OWNERS b/vespaclient/OWNERS index 123437e2758..e030acdbc5b 100644 --- a/vespaclient/OWNERS +++ b/vespaclient/OWNERS @@ -1 +1 @@ -dybis +freva diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp index fe0880d9ea1..e708e623592 100644 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/defaults.h> -#include <vector> -#include <string> +#include <thread> #include <iostream> #include <algorithm> -#include <cstdio> #include <dirent.h> #include <unistd.h> @@ -189,7 +187,7 @@ Application::Main() if (scanInbox() && findOutboxes()) { moveLinks(); } else { - FastOS_Thread::Sleep(200); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); } } } diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp index 7ea270eb306..b1be394b65f 100644 --- a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp +++ b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - #include <vespa/document/util/stringutil.h> #include <vespa/fnet/frt/frt.h> #include <vespa/slobrok/sbmirror.h> -#include <iostream> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/programoptions.h> @@ -14,6 +11,8 @@ #include <vespa/config-stor-distribution.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/fastos/app.h> +#include <sstream> +#include <iostream> #include <vespa/log/log.h> LOG_SETUP("vdsstatetool"); diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp index 303d929f992..ce23fcd93cb 100644 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp @@ -1,9 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "application.h" #include <boost/program_options.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/config/common/exceptions.h> -#include "application.h" +#include <vespa/document/base/idstringexception.h> #include <iostream> + #include <vespa/log/log.h> LOG_SETUP("vespadoclocator"); diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp index 030f284c182..af805d461fc 100644 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp @@ -6,8 +6,7 @@ #include <vespa/messagebus/iconfighandler.h> #include <vespa/messagebus/routing/routingspec.h> #include <vespa/vdslib/bucketdistribution.h> -#include <vespa/messagebus/config-messagebus.h> -#include <vespa/config/helper/configgetter.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/config/helper/configgetter.hpp> diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp index e9482c1eddc..baf7c01d631 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp @@ -12,7 +12,7 @@ #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/config/helper/configgetter.hpp> - +#include <vespa/vespalib/util/stringfmt.h> using config::ConfigGetter; using document::DocumenttypesConfig; diff --git a/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore b/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore +++ /dev/null diff --git a/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore b/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore +++ /dev/null diff --git a/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java b/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java new file mode 100644 index 00000000000..f9e803df08f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/TeeInputStream.java @@ -0,0 +1,97 @@ +package com.yahoo.io; + +import java.io.InputStream; +import java.io.OutputStream; +import java.io.IOException; + +/** + * Forwards input from a source InputStream while making a copy of it into an outputstream. + * Note that it also does read-ahead and copies up to 64K of data more than was used. + */ +class TeeInputStream extends InputStream { + final InputStream src; + final OutputStream dst; + + static final int CAPACITY = 65536; + + byte[] buf = new byte[CAPACITY]; + int readPos = 0; + int writePos = 0; + + private int inBuf() { return writePos - readPos; } + + private void fillBuf() throws IOException { + if (readPos == writePos) { + readPos = 0; + writePos = 0; + } + if (readPos * 3 > CAPACITY) { + int had = inBuf(); + System.arraycopy(buf, readPos, buf, 0, had); + readPos = 0; + writePos = had; + } + int wantToRead = CAPACITY - writePos; + if (inBuf() > 0) { + // if we have data already, do not block, read only what is available + wantToRead = Math.min(wantToRead, src.available()); + } + if (wantToRead > 0) { + int got = src.read(buf, writePos, wantToRead); + if (got > 0) { + dst.write(buf, writePos, got); + writePos += got; + } + } + } + + /** Construct a Tee */ + public TeeInputStream(InputStream from, OutputStream to) { + super(); + this.src = from; + this.dst = to; + } + + @Override + public int available() throws IOException { + return inBuf() + src.available(); + } + + @Override + public void close() throws IOException { + src.close(); + dst.close(); + } + + @Override + public int read() throws IOException { + fillBuf(); + if (inBuf() > 0) { + int r = buf[readPos++]; + return r & 0xff; + } + return -1; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len <= 0) { + return 0; + } + fillBuf(); + int had = inBuf(); + if (had > 0) { + len = Math.min(len, had); + System.arraycopy(buf, readPos, b, off, len); + readPos += len; + return len; + } + return -1; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java index 65201a02bae..7b93429acdb 100644 --- a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java +++ b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java @@ -28,21 +28,32 @@ public class NamedReader extends Reader { public Reader getReader() { return reader; } /** Returns the name */ - public @Override String toString() { + @Override + public String toString() { return name; } // The rest is reader method implementations which delegates to the wrapped reader - public @Override int read(java.nio.CharBuffer charBuffer) throws java.io.IOException { return reader.read(charBuffer); } - public @Override int read() throws java.io.IOException { return reader.read(); } - public @Override int read(char[] chars) throws java.io.IOException { return reader.read(chars); } - public @Override int read(char[] chars, int i, int i1) throws java.io.IOException { return reader.read(chars,i,i1); } - public @Override long skip(long l) throws java.io.IOException { return reader.skip(l); } - public @Override boolean ready() throws java.io.IOException { return reader.ready(); } - public @Override boolean markSupported() { return reader.markSupported(); } - public @Override void mark(int i) throws java.io.IOException { reader.mark(i); } - public @Override void reset() throws java.io.IOException { reader.reset(); } - public @Override void close() throws java.io.IOException { reader.close(); } + @Override + public int read(java.nio.CharBuffer charBuffer) throws java.io.IOException { return reader.read(charBuffer); } + @Override + public int read() throws java.io.IOException { return reader.read(); } + @Override + public int read(char[] chars) throws java.io.IOException { return reader.read(chars); } + @Override + public int read(char[] chars, int i, int i1) throws java.io.IOException { return reader.read(chars,i,i1); } + @Override + public long skip(long l) throws java.io.IOException { return reader.skip(l); } + @Override + public boolean ready() throws java.io.IOException { return reader.ready(); } + @Override + public boolean markSupported() { return reader.markSupported(); } + @Override + public void mark(int i) throws java.io.IOException { reader.mark(i); } + @Override + public void reset() throws java.io.IOException { reader.reset(); } + @Override + public void close() throws java.io.IOException { reader.close(); } /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ public static void closeAll(List<NamedReader> readers) { diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java index 6f381b40cd7..a54e699d777 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Process.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -42,8 +42,10 @@ public final class Process { */ public static void logAndDie(String message, Throwable thrown, boolean dumpThreads) { try { - if (dumpThreads) + if (dumpThreads) { + log.log(Level.INFO, "About to shut down."); dumpThreads(); + } if (thrown != null) log.log(Level.SEVERE, message, thrown); else @@ -60,22 +62,17 @@ public final class Process { } - private static void dumpThreads() { + public static void dumpThreads() { try { - log.log(Level.INFO, "About to shut down. Commencing full thread dump for diagnosis."); + log.log(Level.INFO, "Commencing full thread dump for diagnosis."); Map<Thread, StackTraceElement[]> allStackTraces = Thread.getAllStackTraces(); for (Map.Entry<Thread, StackTraceElement[]> e : allStackTraces.entrySet()) { Thread t = e.getKey(); StackTraceElement[] stack = e.getValue(); StringBuilder forOneThread = new StringBuilder(); - int initLen; forOneThread.append("Stack for thread: ").append(t.getName()).append(": "); - initLen = forOneThread.length(); for (StackTraceElement s : stack) { - if (forOneThread.length() > initLen) { - forOneThread.append(" "); - } - forOneThread.append(s.toString()); + forOneThread.append('\n').append(s.toString()); } log.log(Level.INFO, forOneThread.toString()); } diff --git a/vespajlib/src/main/java/com/yahoo/rmi/.gitignore b/vespajlib/src/main/java/com/yahoo/rmi/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespajlib/src/main/java/com/yahoo/rmi/.gitignore +++ /dev/null diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index a4b1a02f95c..f007db48001 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -156,7 +156,7 @@ public class IndexedTensor implements Tensor { @Override public Map<TensorAddress, Double> cells() { if (dimensionSizes.dimensions() == 0) - return Collections.singletonMap(TensorAddress.empty, values[0]); + return Collections.singletonMap(TensorAddress.of(), values[0]); ImmutableMap.Builder<TensorAddress, Double> builder = new ImmutableMap.Builder<>(); Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, values.length); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index d62e7575358..30eeba8cc4f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -22,8 +22,6 @@ import java.util.Set; @Beta public abstract class TensorAddress implements Comparable<TensorAddress> { - public static final TensorAddress empty = new TensorAddress.Builder(TensorType.empty).build(); - public static TensorAddress of(String[] labels) { return new StringTensorAddress(labels); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index d95feb29af4..6d0703b54d1 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -50,7 +50,7 @@ public class Generate extends PrimitiveTensorFunction { @Override public TensorFunction replaceArguments(List<TensorFunction> arguments) { - if ( arguments.size() != 1) + if ( arguments.size() != 0) throw new IllegalArgumentException("Generate must have 0 arguments, got " + arguments.size()); return this; } diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java index c688d5f9722..1b3d690cb2c 100644 --- a/vespajlib/src/main/java/com/yahoo/text/XML.java +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -25,7 +25,7 @@ import org.xml.sax.SAXParseException; * @author Bjorn Borud * @author Vegard Havdal * @author bratseth - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class XML { /** diff --git a/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java b/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java new file mode 100644 index 00000000000..ac438d8645b --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/TeeInputStreamTest.java @@ -0,0 +1,100 @@ +package com.yahoo.io; + +import java.io.*; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author arnej + */ +public class TeeInputStreamTest { + + @Test + public void testSimpleInput() throws IOException { + byte[] input = "very simple input".getBytes(StandardCharsets.UTF_8); + ByteArrayInputStream in = new ByteArrayInputStream(input); + ByteArrayOutputStream gotten = new ByteArrayOutputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + TeeInputStream tee = new TeeInputStream(in, gotten); + int b = tee.read(); + assertThat(b, is((int)'v')); + output.write(b); + assertThat(gotten.toString(), is("very simple input")); + for (int i = 0; i < 16; i++) { + b = tee.read(); + // System.out.println("got["+i+"]: "+(char)b); + assertThat(b, is(greaterThan(0))); + output.write(b); + } + assertThat(tee.read(), is(-1)); + assertThat(gotten.toString(), is("very simple input")); + assertThat(output.toString(), is("very simple input")); + } + + private class Generator implements Runnable { + private OutputStream dst; + public Generator(OutputStream dst) { this.dst = dst; } + @Override + public void run() { + for (int i = 0; i < 123456789; i++) { + int b = i & 0x7f; + if (b < 32) continue; + if (b > 126) b = '\n'; + try { + dst.write(b); + } catch (IOException e) { + return; + } + } + } + } + + @Test + public void testPipedInput() throws IOException { + PipedOutputStream input = new PipedOutputStream(); + PipedInputStream in = new PipedInputStream(input); + ByteArrayOutputStream gotten = new ByteArrayOutputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + TeeInputStream tee = new TeeInputStream(in, gotten); + input.write("first input".getBytes(StandardCharsets.UTF_8)); + int b = tee.read(); + assertThat(b, is((int)'f')); + output.write(b); + assertThat(gotten.toString(), is("first input")); + input.write(" second input".getBytes(StandardCharsets.UTF_8)); + b = tee.read(); + assertThat(b, is((int)'i')); + output.write(b); + assertThat(gotten.toString(), is("first input second input")); + new Thread(new Generator(input)).start(); + b = tee.read(); + assertThat(b, is((int)'r')); + output.write(b); + byte[] ba = new byte[9]; + for (int i = 0; i < 12345; i++) { + b = tee.read(); + // System.out.println("got["+i+"]: "+(char)b); + assertThat(b, is(greaterThan(0))); + output.write(b); + int l = tee.read(ba); + assertThat(l, is(greaterThan(0))); + output.write(ba, 0, l); + l = tee.read(ba, 3, 3); + assertThat(l, is(greaterThan(0))); + output.write(ba, 3, l); + } + tee.close(); + String got = gotten.toString(); + // System.out.println("got length: "+got.length()); + // System.out.println("output length: "+output.toString().length()); + // System.out.println("got: "+got); + assertThat(got.length(), is(greaterThan(34567))); + assertTrue(got.startsWith(output.toString())); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java index 09b6e79a101..51d2de6c11f 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java @@ -34,7 +34,7 @@ public class IndexedTensorTestCase { @Test public void testSingleValue() { - Tensor singleValue = Tensor.Builder.of(TensorType.empty).cell(TensorAddress.empty, 3.5).build(); + Tensor singleValue = Tensor.Builder.of(TensorType.empty).cell(TensorAddress.of(), 3.5).build(); assertTrue(singleValue instanceof IndexedTensor); assertEquals("{3.5}", singleValue.toString()); Tensor singleValueFromString = Tensor.from(TensorType.empty, "{3.5}"); diff --git a/vespalib/src/tests/alignedmemory/alignedmemory_test.cpp b/vespalib/src/tests/alignedmemory/alignedmemory_test.cpp index 73814d3284e..3d55e15e8e6 100644 --- a/vespalib/src/tests/alignedmemory/alignedmemory_test.cpp +++ b/vespalib/src/tests/alignedmemory/alignedmemory_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("alignedmemory_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/atomic/atomic_bench.cpp b/vespalib/src/tests/atomic/atomic_bench.cpp index 003ef25ba45..6294f956507 100644 --- a/vespalib/src/tests/atomic/atomic_bench.cpp +++ b/vespalib/src/tests/atomic/atomic_bench.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/atomic.h> +#include <vespa/fastos/thread.h> #include <vector> #include <algorithm> #include <sstream> diff --git a/vespalib/src/tests/atomic/atomic_test.cpp b/vespalib/src/tests/atomic/atomic_test.cpp index c08325d8f30..09fc99edeb9 100644 --- a/vespalib/src/tests/atomic/atomic_test.cpp +++ b/vespalib/src/tests/atomic/atomic_test.cpp @@ -1,13 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("atomic_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/atomic.h> +#include <vespa/fastos/thread.h> #include <vector> #include <algorithm> #include <sstream> +#include <vespa/log/log.h> +LOG_SETUP("atomic_test"); class Test : public vespalib::TestApp { diff --git a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp index 42f6af0e1ea..ac6a57358d6 100644 --- a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp +++ b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/benchmark_timer.h> @@ -9,7 +8,7 @@ TEST("require that the benchmark timer can be used as advertised") { BenchmarkTimer timer(1.0); while (timer.has_budget()) { timer.before(); - FastOS_Thread::Sleep(5); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); timer.after(); } EXPECT_TRUE(timer.min_time() >= 0.0); @@ -18,15 +17,15 @@ TEST("require that the benchmark timer can be used as advertised") { TEST("require that the benchmark timer all-in-one benchmarking works") { uint32_t sleep_time = 5; - double t = BenchmarkTimer::benchmark([sleep_time](){FastOS_Thread::Sleep(sleep_time);}, 1.0); + double t = BenchmarkTimer::benchmark([sleep_time](){std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time));}, 1.0); fprintf(stderr, "5 ms sleep takes: %g ms\n", t * 1000.0); } TEST("require that the benchmark timer all-in-one benchmarking with baseline works") { uint32_t work_time = 10; uint32_t baseline_time = 5; - double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time);}, - [&](){FastOS_Thread::Sleep(baseline_time);}, 1.0); + double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time));}, + [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 1.0); fprintf(stderr, "10 ms sleep - 5 ms sleep takes: %g ms\n", t * 1000.0); } @@ -34,8 +33,8 @@ TEST("require that the benchmark timer all-in-one benchmarking with baseline and uint32_t work_time = 2; uint32_t baseline_time = 1; uint32_t loop_cnt = 0; - double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time); ++loop_cnt;}, - [&](){FastOS_Thread::Sleep(baseline_time);}, 7, 0.0); + double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time)); ++loop_cnt;}, + [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 7, 0.0); EXPECT_EQUAL(loop_cnt, 7u); fprintf(stderr, "2 ms sleep - 1 ms sleep takes: %g ms\n", t * 1000.0); } diff --git a/vespalib/src/tests/component/component.cpp b/vespalib/src/tests/component/component.cpp index ee21f816588..0c7cb6a6485 100644 --- a/vespalib/src/tests/component/component.cpp +++ b/vespalib/src/tests/component/component.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("component_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/delegatelist/delegatelist.cpp b/vespalib/src/tests/delegatelist/delegatelist.cpp index 11dd4d7d9e2..6a9751440a6 100644 --- a/vespalib/src/tests/delegatelist/delegatelist.cpp +++ b/vespalib/src/tests/delegatelist/delegatelist.cpp @@ -3,10 +3,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/delegatelist.hpp> #include <vespa/vespalib/util/guard.h> - -#include <memory> -#include <algorithm> -#include <vector> +#include <vespa/fastos/thread.h> #include <queue> #include <vespa/log/log.h> diff --git a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp index 07c3db8e75c..c241707a92e 100644 --- a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp +++ b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> #include <iostream> +#include <vector> using namespace vespalib; using vespalib::hwaccelrated::IAccelrated; diff --git a/vespalib/src/tests/exception_classes/exception_classes_test.cpp b/vespalib/src/tests/exception_classes/exception_classes_test.cpp index 946c3fa32e4..a418cb421ed 100644 --- a/vespalib/src/tests/exception_classes/exception_classes_test.cpp +++ b/vespalib/src/tests/exception_classes/exception_classes_test.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace vespalib; diff --git a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp index f8136fe2a10..6b295497744 100644 --- a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/backtrace.h> #include <thread> using namespace vespalib; @@ -108,4 +109,32 @@ TEST_F("require that task limit can be decreased", Fixture(3, 3)) f.blockedExecuteAndWaitUntilFinished(); } +vespalib::string get_worker_stack_trace(BlockingThreadStackExecutor &executor) { + struct StackTraceTask : public Executor::Task { + vespalib::string &trace; + explicit StackTraceTask(vespalib::string &t) : trace(t) {} + void run() override { trace = getStackTrace(0); } + }; + vespalib::string trace; + executor.execute(std::make_unique<StackTraceTask>(trace)); + executor.sync(); + return trace; +} + +VESPA_THREAD_STACK_TAG(my_stack_tag); + +TEST_F("require that executor has appropriate default thread stack tag", BlockingThreadStackExecutor(1, 128*1024, 10)) { + vespalib::string trace = get_worker_stack_trace(f1); + if (!EXPECT_TRUE(trace.find("unnamed_blocking_executor") != vespalib::string::npos)) { + fprintf(stderr, "%s\n", trace.c_str()); + } +} + +TEST_F("require that executor thread stack tag can be set", BlockingThreadStackExecutor(1, 128*1024, 10, my_stack_tag)) { + vespalib::string trace = get_worker_stack_trace(f1); + if (!EXPECT_TRUE(trace.find("my_stack_tag") != vespalib::string::npos)) { + fprintf(stderr, "%s\n", trace.c_str()); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index cb998a4323c..143a45c5f3f 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/backtrace.h> using namespace vespalib; @@ -157,4 +158,32 @@ TEST_MT_F("require that threads can wait for a specific task count", 7, WaitStat } } +vespalib::string get_worker_stack_trace(ThreadStackExecutor &executor) { + struct StackTraceTask : public Executor::Task { + vespalib::string &trace; + explicit StackTraceTask(vespalib::string &t) : trace(t) {} + void run() override { trace = getStackTrace(0); } + }; + vespalib::string trace; + executor.execute(std::make_unique<StackTraceTask>(trace)); + executor.sync(); + return trace; +} + +VESPA_THREAD_STACK_TAG(my_stack_tag); + +TEST_F("require that executor has appropriate default thread stack tag", ThreadStackExecutor(1, 128*1024)) { + vespalib::string trace = get_worker_stack_trace(f1); + if (!EXPECT_TRUE(trace.find("unnamed_nonblocking_executor") != vespalib::string::npos)) { + fprintf(stderr, "%s\n", trace.c_str()); + } +} + +TEST_F("require that executor thread stack tag can be set", ThreadStackExecutor(1, 128*1024, my_stack_tag)) { + vespalib::string trace = get_worker_stack_trace(f1); + if (!EXPECT_TRUE(trace.find("my_stack_tag") != vespalib::string::npos)) { + fprintf(stderr, "%s\n", trace.c_str()); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/false/false.cpp b/vespalib/src/tests/false/false.cpp index 5c93d402086..003477f2469 100644 --- a/vespalib/src/tests/false/false.cpp +++ b/vespalib/src/tests/false/false.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("false_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/gencnt/gencnt_test.cpp b/vespalib/src/tests/gencnt/gencnt_test.cpp index af192dd1a78..e313901d65c 100644 --- a/vespalib/src/tests/gencnt/gencnt_test.cpp +++ b/vespalib/src/tests/gencnt/gencnt_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("gencnt_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/left_right_heap/left_right_heap_test.cpp b/vespalib/src/tests/left_right_heap/left_right_heap_test.cpp index f06d0534ef6..babeb23ce82 100644 --- a/vespalib/src/tests/left_right_heap/left_right_heap_test.cpp +++ b/vespalib/src/tests/left_right_heap/left_right_heap_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/left_right_heap.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp index 61801c1d3b2..881583f02c9 100644 --- a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp +++ b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/slaveproc.h> diff --git a/vespalib/src/tests/net/socket/socket_client.cpp b/vespalib/src/tests/net/socket/socket_client.cpp index fd4abd2813d..bf8a9fe9e6d 100644 --- a/vespalib/src/tests/net/socket/socket_client.cpp +++ b/vespalib/src/tests/net/socket/socket_client.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket.h> diff --git a/vespalib/src/tests/net/socket/socket_server.cpp b/vespalib/src/tests/net/socket/socket_server.cpp index b6af5fa120b..374a8f57b83 100644 --- a/vespalib/src/tests/net/socket/socket_server.cpp +++ b/vespalib/src/tests/net/socket/socket_server.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> #include <vespa/vespalib/net/socket.h> diff --git a/vespalib/src/tests/priority_queue/priority_queue_test.cpp b/vespalib/src/tests/priority_queue/priority_queue_test.cpp index fd37215db1c..bf43066caec 100644 --- a/vespalib/src/tests/priority_queue/priority_queue_test.cpp +++ b/vespalib/src/tests/priority_queue/priority_queue_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("priority_queue_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/random/friendfinder.cpp b/vespalib/src/tests/random/friendfinder.cpp index 30c0dd33612..7bf390ef083 100644 --- a/vespalib/src/tests/random/friendfinder.cpp +++ b/vespalib/src/tests/random/friendfinder.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/util/random.h> -#include <stdio.h> #include <vespa/vespalib/stllike/string.h> #include <cmath> +#include <vector> int main(int argc, char **argv) { diff --git a/vespalib/src/tests/referencecounter/referencecounter_test.cpp b/vespalib/src/tests/referencecounter/referencecounter_test.cpp index 6f510f1f2a5..f5ed5696057 100644 --- a/vespalib/src/tests/referencecounter/referencecounter_test.cpp +++ b/vespalib/src/tests/referencecounter/referencecounter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("referencecounter_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/sha1/rfc_sha1.cpp b/vespalib/src/tests/sha1/rfc_sha1.cpp index f0f2acde060..a656b81f5a9 100644 --- a/vespalib/src/tests/sha1/rfc_sha1.cpp +++ b/vespalib/src/tests/sha1/rfc_sha1.cpp @@ -37,7 +37,6 @@ * a multiple of the size of an 8-bit character. * */ -#include <vespa/fastos/fastos.h> #include "rfc_sha1.h" /* * Define the SHA1 circular left shift macro diff --git a/vespalib/src/tests/signalhandler/signalhandler_test.cpp b/vespalib/src/tests/signalhandler/signalhandler_test.cpp index 2aa0445a339..c52e084c184 100644 --- a/vespalib/src/tests/signalhandler/signalhandler_test.cpp +++ b/vespalib/src/tests/signalhandler/signalhandler_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("signalhandler_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/signalhandler/victim.cpp b/vespalib/src/tests/signalhandler/victim.cpp index fe42489006c..f0a96f2c055 100644 --- a/vespalib/src/tests/signalhandler/victim.cpp +++ b/vespalib/src/tests/signalhandler/victim.cpp @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/vespalib/util/signalhandler.h> +#include <cstdio> +#include <unistd.h> using vespalib::SignalHandler; diff --git a/vespalib/src/tests/slime/json_slime_benchmark.cpp b/vespalib/src/tests/slime/json_slime_benchmark.cpp index e818b2d4c34..2a1a806e177 100644 --- a/vespalib/src/tests/slime/json_slime_benchmark.cpp +++ b/vespalib/src/tests/slime/json_slime_benchmark.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/testkit/test_kit.h> #include <iostream> diff --git a/vespalib/src/tests/slime/slime_json_format_test.cpp b/vespalib/src/tests/slime/slime_json_format_test.cpp index c87f0d9d4af..27c92d3df8f 100644 --- a/vespalib/src/tests/slime/slime_json_format_test.cpp +++ b/vespalib/src/tests/slime/slime_json_format_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <iostream> diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index e15bb26be53..7c3f6096ef1 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("slime_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/stllike/lookup_benchmark.cpp b/vespalib/src/tests/stllike/lookup_benchmark.cpp index 9358f189f0a..2c20b53292b 100644 --- a/vespalib/src/tests/stllike/lookup_benchmark.cpp +++ b/vespalib/src/tests/stllike/lookup_benchmark.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <stddef.h> #include <stdlib.h> #include <stdio.h> diff --git a/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp b/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp index 5497017947e..db06bf5a349 100644 --- a/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp +++ b/vespalib/src/tests/stllike/uniq_by_sort_map_hash.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <stddef.h> #include <stdlib.h> #include <stdio.h> diff --git a/vespalib/src/tests/stllike/vector_map_test.cpp b/vespalib/src/tests/stllike/vector_map_test.cpp index 76c80173b0b..31294d0c577 100644 --- a/vespalib/src/tests/stllike/vector_map_test.cpp +++ b/vespalib/src/tests/stllike/vector_map_test.cpp @@ -3,7 +3,6 @@ #include <vespa/log/log.h> LOG_SETUP("vector_map_test"); -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/stllike/vector_map.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/testapp-debug/debugtest.cpp b/vespalib/src/tests/testapp-debug/debugtest.cpp index a3c22d2f04a..1189b39d782 100644 --- a/vespalib/src/tests/testapp-debug/debugtest.cpp +++ b/vespalib/src/tests/testapp-debug/debugtest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; diff --git a/vespalib/src/tests/testapp-debug/testapp-debug.cpp b/vespalib/src/tests/testapp-debug/testapp-debug.cpp index 3081048ea7a..2a845e2515c 100644 --- a/vespalib/src/tests/testapp-debug/testapp-debug.cpp +++ b/vespalib/src/tests/testapp-debug/testapp-debug.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; diff --git a/vespalib/src/tests/testapp-generic/testapp-generic.cpp b/vespalib/src/tests/testapp-generic/testapp-generic.cpp index 6a71ea4532c..b96c3bc5ad8 100644 --- a/vespalib/src/tests/testapp-generic/testapp-generic.cpp +++ b/vespalib/src/tests/testapp-generic/testapp-generic.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <stdexcept> diff --git a/vespalib/src/tests/testapp-main/testapp-main_test.cpp b/vespalib/src/tests/testapp-main/testapp-main_test.cpp index c252d8fecc5..4bae14cb00e 100644 --- a/vespalib/src/tests/testapp-main/testapp-main_test.cpp +++ b/vespalib/src/tests/testapp-main/testapp-main_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> void subtest() { diff --git a/vespalib/src/tests/testapp-state/expect.txt b/vespalib/src/tests/testapp-state/expect.txt index 87b7721907e..0cd267e8245 100644 --- a/vespalib/src/tests/testapp-state/expect.txt +++ b/vespalib/src/tests/testapp-state/expect.txt @@ -1,22 +1,22 @@ - STATE[0]: 'foo' (statetest.cpp:26) - STATE[1]: 'bar' (statetest.cpp:29) - STATE[0]: 'foo' (statetest.cpp:26) - STATE[2]: 'baz' (statetest.cpp:32) - STATE[1]: 'bar' (statetest.cpp:29) - STATE[0]: 'foo' (statetest.cpp:26) - STATE[1]: 'bar' (statetest.cpp:29) - STATE[0]: 'foo' (statetest.cpp:26) - STATE[0]: 'foo' (statetest.cpp:26) - STATE[0]: 'testSomething()' (statetest.cpp:42) - STATE[1]: 'testInner()' (statetest.cpp:13) - STATE[0]: 'testSomething()' (statetest.cpp:42) - STATE[0]: 'testSomething()' (statetest.cpp:42) - STATE[0]: 'something else' (statetest.cpp:45) - STATE[1]: 'testSomethingElse()' (statetest.cpp:47) - STATE[0]: 'something else' (statetest.cpp:45) - STATE[2]: 'testInner()' (statetest.cpp:19) - STATE[1]: 'testSomethingElse()' (statetest.cpp:47) - STATE[0]: 'something else' (statetest.cpp:45) - STATE[1]: 'testSomethingElse()' (statetest.cpp:47) - STATE[0]: 'something else' (statetest.cpp:45) - STATE[0]: 'something else' (statetest.cpp:45) + STATE[0]: 'foo' (statetest.cpp:25) + STATE[1]: 'bar' (statetest.cpp:28) + STATE[0]: 'foo' (statetest.cpp:25) + STATE[2]: 'baz' (statetest.cpp:31) + STATE[1]: 'bar' (statetest.cpp:28) + STATE[0]: 'foo' (statetest.cpp:25) + STATE[1]: 'bar' (statetest.cpp:28) + STATE[0]: 'foo' (statetest.cpp:25) + STATE[0]: 'foo' (statetest.cpp:25) + STATE[0]: 'testSomething()' (statetest.cpp:41) + STATE[1]: 'testInner()' (statetest.cpp:12) + STATE[0]: 'testSomething()' (statetest.cpp:41) + STATE[0]: 'testSomething()' (statetest.cpp:41) + STATE[0]: 'something else' (statetest.cpp:44) + STATE[1]: 'testSomethingElse()' (statetest.cpp:46) + STATE[0]: 'something else' (statetest.cpp:44) + STATE[2]: 'testInner()' (statetest.cpp:18) + STATE[1]: 'testSomethingElse()' (statetest.cpp:46) + STATE[0]: 'something else' (statetest.cpp:44) + STATE[1]: 'testSomethingElse()' (statetest.cpp:46) + STATE[0]: 'something else' (statetest.cpp:44) + STATE[0]: 'something else' (statetest.cpp:44) diff --git a/vespalib/src/tests/testapp-state/statetest.cpp b/vespalib/src/tests/testapp-state/statetest.cpp index 4ca9fc4259e..1f57c51c39d 100644 --- a/vespalib/src/tests/testapp-state/statetest.cpp +++ b/vespalib/src/tests/testapp-state/statetest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; diff --git a/vespalib/src/tests/testapp-state/testapp-state.cpp b/vespalib/src/tests/testapp-state/testapp-state.cpp index cf6ced914d4..a6b2c23d5ec 100644 --- a/vespalib/src/tests/testapp-state/testapp-state.cpp +++ b/vespalib/src/tests/testapp-state/testapp-state.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; diff --git a/vespalib/src/tests/testkit-mt/testkit-mt_test.cpp b/vespalib/src/tests/testkit-mt/testkit-mt_test.cpp index 336a748773e..b29e498002b 100644 --- a/vespalib/src/tests/testkit-mt/testkit-mt_test.cpp +++ b/vespalib/src/tests/testkit-mt/testkit-mt_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> using namespace vespalib; diff --git a/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp b/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp index 6234b6f6308..386e4b26885 100644 --- a/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp +++ b/vespalib/src/tests/testkit-testhook/testkit-testhook_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/vespalib/src/tests/text/lowercase/lowercase_test.cpp b/vespalib/src/tests/text/lowercase/lowercase_test.cpp index 0d2514a0a21..d6de7d685da 100644 --- a/vespalib/src/tests/text/lowercase/lowercase_test.cpp +++ b/vespalib/src/tests/text/lowercase/lowercase_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("lowercase_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp b/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp index 1c88cb6587e..7c494bac751 100644 --- a/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp +++ b/vespalib/src/tests/text/stringtokenizer/stringtokenizer_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("stringtokenizer_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/text/utf8/make_url.cpp b/vespalib/src/tests/text/utf8/make_url.cpp index 29b0cf5dcc2..1c46e3b47f0 100644 --- a/vespalib/src/tests/text/utf8/make_url.cpp +++ b/vespalib/src/tests/text/utf8/make_url.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/text/utf8.h> void printCodepoint(unsigned long codepoint) diff --git a/vespalib/src/tests/text/utf8/utf8_test.cpp b/vespalib/src/tests/text/utf8/utf8_test.cpp index fd58dbfa044..2976313ba95 100644 --- a/vespalib/src/tests/text/utf8/utf8_test.cpp +++ b/vespalib/src/tests/text/utf8/utf8_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("utf8_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/vespalib/src/tests/time/time_box_test.cpp b/vespalib/src/tests/time/time_box_test.cpp index 478d749e15d..32cf1eb66b3 100644 --- a/vespalib/src/tests/time/time_box_test.cpp +++ b/vespalib/src/tests/time/time_box_test.cpp @@ -10,13 +10,13 @@ TEST("require that long-lived timebox returns falling time left numbers") { double timeLeft = box.timeLeft(); EXPECT_TRUE(timeLeft <= last_timeLeft); last_timeLeft = timeLeft; - FastOS_Thread::Sleep(10); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } TEST("require that short-lived timebox times out") { vespalib::TimeBox box(0.125); - FastOS_Thread::Sleep(150); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); EXPECT_FALSE(box.hasTimeLeft()); EXPECT_EQUAL(box.timeLeft(), 0.0); } @@ -27,7 +27,7 @@ TEST("require that short-lived timebox always returns at least minimum time") { double timeLeft = box.timeLeft(); EXPECT_TRUE(timeLeft <= 0.250); EXPECT_TRUE(timeLeft >= 0.125); - FastOS_Thread::Sleep(30); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); } EXPECT_FALSE(box.hasTimeLeft()); EXPECT_EQUAL(box.timeLeft(), 0.125); diff --git a/vespalib/src/tests/time_tracker/time_tracker_test.cpp b/vespalib/src/tests/time_tracker/time_tracker_test.cpp index 0f1bf57c378..8b3b5197d5d 100644 --- a/vespalib/src/tests/time_tracker/time_tracker_test.cpp +++ b/vespalib/src/tests/time_tracker/time_tracker_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/time_tracker.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/vespalib/src/tests/trace/trace.cpp b/vespalib/src/tests/trace/trace.cpp index 7e3ead52206..2bd5d8d00a6 100644 --- a/vespalib/src/tests/trace/trace.cpp +++ b/vespalib/src/tests/trace/trace.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("trace_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/trace/trace_serialization.cpp b/vespalib/src/tests/trace/trace_serialization.cpp index a146f4d0ae6..f3b60c3596d 100644 --- a/vespalib/src/tests/trace/trace_serialization.cpp +++ b/vespalib/src/tests/trace/trace_serialization.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("trace_test"); #include <vespa/vespalib/testkit/test_kit.h> diff --git a/vespalib/src/tests/traits/traits_test.cpp b/vespalib/src/tests/traits/traits_test.cpp index 82824d8c2ee..61725332df3 100644 --- a/vespalib/src/tests/traits/traits_test.cpp +++ b/vespalib/src/tests/traits/traits_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/traits.h> #include <vespa/vespalib/util/arrayqueue.hpp> diff --git a/vespalib/src/tests/true/true.cpp b/vespalib/src/tests/true/true.cpp index d052f3ca4e4..ec88a83d2c2 100644 --- a/vespalib/src/tests/true/true.cpp +++ b/vespalib/src/tests/true/true.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("true_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index a11cff02d22..a8dd34ffd3d 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("generation_handler_stress_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/tests/zcurve/zcurve_test.cpp b/vespalib/src/tests/zcurve/zcurve_test.cpp index 41bccf23ccf..9807ea0a206 100644 --- a/vespalib/src/tests/zcurve/zcurve_test.cpp +++ b/vespalib/src/tests/zcurve/zcurve_test.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/log/log.h> LOG_SETUP("zcurve_test"); #include <vespa/vespalib/testkit/testapp.h> diff --git a/vespalib/src/vespa/vespalib/data/input.cpp b/vespalib/src/vespa/vespalib/data/input.cpp index d0dc70eb107..d27e3d68c67 100644 --- a/vespalib/src/vespa/vespalib/data/input.cpp +++ b/vespalib/src/vespa/vespalib/data/input.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "input.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/input_reader.cpp b/vespalib/src/vespa/vespalib/data/input_reader.cpp index b71935c795b..7b70acf0cef 100644 --- a/vespalib/src/vespa/vespalib/data/input_reader.cpp +++ b/vespalib/src/vespa/vespalib/data/input_reader.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "input_reader.h" #include "input.h" diff --git a/vespalib/src/vespa/vespalib/data/lz4_input_decoder.cpp b/vespalib/src/vespa/vespalib/data/lz4_input_decoder.cpp index 39407449105..ed15af58f2c 100644 --- a/vespalib/src/vespa/vespalib/data/lz4_input_decoder.cpp +++ b/vespalib/src/vespa/vespalib/data/lz4_input_decoder.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "lz4_input_decoder.h" #include <lz4frame.h> +#include <cassert> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/lz4_input_decoder.h b/vespalib/src/vespa/vespalib/data/lz4_input_decoder.h index f711aef30b5..5f03529a945 100644 --- a/vespalib/src/vespa/vespalib/data/lz4_input_decoder.h +++ b/vespalib/src/vespa/vespalib/data/lz4_input_decoder.h @@ -3,6 +3,7 @@ #pragma once #include "input.h" +#include <vector> struct LZ4F_dctx_s; diff --git a/vespalib/src/vespa/vespalib/data/lz4_output_encoder.cpp b/vespalib/src/vespa/vespalib/data/lz4_output_encoder.cpp index 60f8546bde1..3ae16524a8e 100644 --- a/vespalib/src/vespa/vespalib/data/lz4_output_encoder.cpp +++ b/vespalib/src/vespa/vespalib/data/lz4_output_encoder.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "lz4_output_encoder.h" #include <lz4frame.h> +#include <cassert> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/lz4_output_encoder.h b/vespalib/src/vespa/vespalib/data/lz4_output_encoder.h index 6936a9a0169..98005eacf60 100644 --- a/vespalib/src/vespa/vespalib/data/lz4_output_encoder.h +++ b/vespalib/src/vespa/vespalib/data/lz4_output_encoder.h @@ -3,6 +3,7 @@ #pragma once #include "output.h" +#include <vector> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/memory.cpp b/vespalib/src/vespa/vespalib/data/memory.cpp index 71c8c6c8ced..3f76b86ebae 100644 --- a/vespalib/src/vespa/vespalib/data/memory.cpp +++ b/vespalib/src/vespa/vespalib/data/memory.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "memory.h" #include <vespa/vespalib/util/stringfmt.h> diff --git a/vespalib/src/vespa/vespalib/data/memory_input.cpp b/vespalib/src/vespa/vespalib/data/memory_input.cpp index 485b438f33e..01d94f7f1b6 100644 --- a/vespalib/src/vespa/vespalib/data/memory_input.cpp +++ b/vespalib/src/vespa/vespalib/data/memory_input.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "memory_input.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/output.cpp b/vespalib/src/vespa/vespalib/data/output.cpp index 27489da4179..667b5ed763f 100644 --- a/vespalib/src/vespa/vespalib/data/output.cpp +++ b/vespalib/src/vespa/vespalib/data/output.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "output.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/output_writer.cpp b/vespalib/src/vespa/vespalib/data/output_writer.cpp index a04aba9c733..935af47bdbe 100644 --- a/vespalib/src/vespa/vespalib/data/output_writer.cpp +++ b/vespalib/src/vespa/vespalib/data/output_writer.cpp @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "output_writer.h" #include "output.h" -#include <stdarg.h> -#include <stdio.h> -#include <assert.h> +#include <cstdarg> +#include <cstdio> +#include <cassert> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/writable_memory.cpp b/vespalib/src/vespa/vespalib/data/writable_memory.cpp index b94c54e13cd..d8be1b6367f 100644 --- a/vespalib/src/vespa/vespalib/data/writable_memory.cpp +++ b/vespalib/src/vespa/vespalib/data/writable_memory.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "writable_memory.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index c9778971efd..2e3994d329f 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fileutil.h" - -#include <errno.h> -#include <ostream> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/error.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/fastos/file.h> +#include <ostream> +#include <sys/stat.h> #include <vespa/log/log.h> LOG_SETUP(".vespalib.io.fileutil"); diff --git a/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp b/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp index d9108456347..a52012b437e 100644 --- a/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp +++ b/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "mapped_file_input.h" +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/mman.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/net/selector.cpp b/vespalib/src/vespa/vespalib/net/selector.cpp index 01027e5b762..05784cd07f6 100644 --- a/vespalib/src/vespa/vespalib/net/selector.cpp +++ b/vespalib/src/vespa/vespalib/net/selector.cpp @@ -1,7 +1,12 @@ // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "selector.h" +#include <cassert> +#include <cstdlib> +#include <cerrno> +#include <cstring> +#include <unistd.h> +#include <fcntl.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/net/server_socket.cpp b/vespalib/src/vespa/vespalib/net/server_socket.cpp index 9359f5824c0..0a5310bbeb3 100644 --- a/vespalib/src/vespa/vespalib/net/server_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/server_socket.cpp @@ -2,6 +2,8 @@ #include "server_socket.h" #include "socket_spec.h" +#include <sys/stat.h> +#include <dirent.h> #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.server_socket"); diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index ec04ffb8b3b..e37399bdfcc 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -2,7 +2,7 @@ #include "nbostream.hpp" #include "hexdump.h" #include <vespa/vespalib/util/exceptions.h> - +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/testkit/test_macros.h b/vespalib/src/vespa/vespalib/testkit/test_macros.h index 27244c07372..f7532b026ec 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_macros.h +++ b/vespalib/src/vespa/vespalib/testkit/test_macros.h @@ -61,8 +61,8 @@ throw; \ } -#define EXPECT_TRUE(rc) TEST_MASTER.check(rc, __FILE__, __LINE__, TEST_STR(rc), false) -#define EXPECT_FALSE(rc) TEST_MASTER.check(!(rc), __FILE__, __LINE__, TEST_STR(rc), false) +#define EXPECT_TRUE(rc) TEST_MASTER.check(bool(rc), __FILE__, __LINE__, TEST_STR(rc), false) +#define EXPECT_FALSE(rc) TEST_MASTER.check(!(bool(rc)), __FILE__, __LINE__, TEST_STR(rc), false) #define EXPECT_APPROX(a, b, eps) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " ~= ", a, b, vespalib::TestComparators::approx(eps), false) #define EXPECT_NOT_APPROX(a, b, eps) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " !~= ", a, b, vespalib::TestComparators::not_approx(eps), false) #define EXPECT_EQUAL(a, b) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " == ", a, b, vespalib::TestComparators::equal(), false) @@ -76,8 +76,8 @@ #define TEST_ERROR(msg) TEST_MASTER.check(false, __FILE__, __LINE__, msg, false) -#define ASSERT_TRUE(rc) TEST_MASTER.check(rc, __FILE__, __LINE__, TEST_STR(rc), true) -#define ASSERT_FALSE(rc) TEST_MASTER.check(!(rc), __FILE__, __LINE__, TEST_STR(rc), true) +#define ASSERT_TRUE(rc) TEST_MASTER.check(bool(rc), __FILE__, __LINE__, TEST_STR(rc), true) +#define ASSERT_FALSE(rc) TEST_MASTER.check(!(bool(rc)), __FILE__, __LINE__, TEST_STR(rc), true) #define ASSERT_APPROX(a, b, eps) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " ~= ", a, b, vespalib::TestComparators::approx(eps), true) #define ASSERT_NOT_APPROX(a, b, eps) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " !~= ", a, b, vespalib::TestComparators::not_approx(eps), true) #define ASSERT_EQUAL(a, b) TEST_MASTER.compare(__FILE__, __LINE__, TEST_STR(a), TEST_STR(b), " == ", a, b, vespalib::TestComparators::equal(), true) diff --git a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp index bb6214b6011..f52c534498f 100644 --- a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp @@ -4,6 +4,8 @@ namespace vespalib { +VESPA_THREAD_STACK_TAG(unnamed_blocking_executor); + bool BlockingThreadStackExecutor::acceptNewTask(MonitorGuard & guard) { @@ -19,8 +21,15 @@ BlockingThreadStackExecutor::wakeup(MonitorGuard & monitor) monitor.broadcast(); } -BlockingThreadStackExecutor::BlockingThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit) : - ThreadStackExecutorBase(stackSize, taskLimit) +BlockingThreadStackExecutor::BlockingThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit) + : ThreadStackExecutorBase(stackSize, taskLimit, unnamed_blocking_executor) +{ + start(threads); +} + +BlockingThreadStackExecutor::BlockingThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit, + init_fun_t init_function) + : ThreadStackExecutorBase(stackSize, taskLimit, std::move(init_function)) { start(threads); } diff --git a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h index 1274ed7fcea..13d8eece40e 100644 --- a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h +++ b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h @@ -27,6 +27,12 @@ public: * @param taskLimit upper limit on accepted tasks **/ BlockingThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit); + + // same as above, but enables you to specify a custom function + // used to wrap the main loop of all worker threads + BlockingThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit, + init_fun_t init_function); + ~BlockingThreadStackExecutor(); /** diff --git a/vespalib/src/vespa/vespalib/util/buffer.h b/vespalib/src/vespa/vespalib/util/buffer.h index 88fe6359aae..3a43a2c0dc4 100644 --- a/vespalib/src/vespa/vespalib/util/buffer.h +++ b/vespalib/src/vespa/vespalib/util/buffer.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <sys/types.h> +#include <cstddef> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/exception.h b/vespalib/src/vespa/vespalib/util/exception.h index 44911ea5555..05f48295bfa 100644 --- a/vespalib/src/vespa/vespalib/util/exception.h +++ b/vespalib/src/vespa/vespalib/util/exception.h @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/macro.h> #include <vespa/vespalib/util/error.h> -#include <vespa/vespalib/util/stringfmt.h> #include <exception> #define VESPALIB_EXCEPTION_USEBACKTRACES diff --git a/vespalib/src/vespa/vespalib/util/exceptions.h b/vespalib/src/vespa/vespalib/util/exceptions.h index 65d17baf654..ca05dac107f 100644 --- a/vespalib/src/vespa/vespalib/util/exceptions.h +++ b/vespalib/src/vespa/vespalib/util/exceptions.h @@ -13,6 +13,7 @@ #pragma once #include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/stringfmt.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/hashmap.h b/vespalib/src/vespa/vespalib/util/hashmap.h index f357ab6cf04..cbaa9540878 100644 --- a/vespalib/src/vespa/vespalib/util/hashmap.h +++ b/vespalib/src/vespa/vespalib/util/hashmap.h @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <inttypes.h> -#include <string.h> -#include <stdlib.h> - #include "hashmapdata.h" +#include <cstring> +#include <cstdlib> /** * @brief namespace for generic Vespa library diff --git a/vespalib/src/vespa/vespalib/util/hashmapdata.h b/vespalib/src/vespa/vespalib/util/hashmapdata.h index 42110246be2..e887e12d245 100644 --- a/vespalib/src/vespa/vespalib/util/hashmapdata.h +++ b/vespalib/src/vespa/vespalib/util/hashmapdata.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <inttypes.h> +#include <cstdint> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h index 2d940882320..85c8e46dcc2 100644 --- a/vespalib/src/vespa/vespalib/util/memory.h +++ b/vespalib/src/vespa/vespalib/util/memory.h @@ -4,8 +4,8 @@ #pragma once #include <memory> -#include <string.h> -#include <stdlib.h> +#include <cstring> +#include <cstdlib> /// Macro to give you number of elements in a defined array. #define VESPA_NELEMS(a) (sizeof(a)/sizeof(a[0])) diff --git a/vespalib/src/vespa/vespalib/util/random.cpp b/vespalib/src/vespa/vespalib/util/random.cpp index 06bfc1f9c25..2abf67605a8 100644 --- a/vespalib/src/vespa/vespalib/util/random.cpp +++ b/vespalib/src/vespa/vespalib/util/random.cpp @@ -2,6 +2,9 @@ #include "random.h" #include <cmath> +#include <cstring> +#include <ctime> +#include <unistd.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/random.h b/vespalib/src/vespa/vespalib/util/random.h index c4cce3b42d8..f9d2587272d 100644 --- a/vespalib/src/vespa/vespalib/util/random.h +++ b/vespalib/src/vespa/vespalib/util/random.h @@ -1,14 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <stdlib.h> -#include <stdio.h> -#include <inttypes.h> -#include <vespa/fastos/types.h> -#include <vector> -#include <sys/types.h> -#include <unistd.h> -#include <time.h> +#include <cstdint> namespace vespalib { @@ -22,9 +15,9 @@ private: /** * step the random generator once **/ - void iterate(void) { - _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & - UINT64_C(0xFFFFFFFFFFFF); + void iterate() { + _state = (0x5DEECE66Dul * _state + 0xb) & + 0xFFFFFFFFFFFFul; } /** @@ -73,7 +66,7 @@ public: * @brief reset the seed **/ void setSeed(int64_t seed) { - _state = (seed ^ UINT64_C(0x5DEECE66D)) & ((1L << 48) -1); + _state = (seed ^ 0x5DEECE66Dul) & ((1L << 48) -1); }; /** diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h index a44763ec755..315bb79487e 100644 --- a/vespalib/src/vespa/vespalib/util/sync.h +++ b/vespalib/src/vespa/vespalib/util/sync.h @@ -34,7 +34,6 @@ public: }; #endif - /** * @brief A Lock is a synchronization primitive used to ensure mutual * exclusion. diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutor.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutor.cpp index 10fbd129ce6..aff52968c7c 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutor.cpp @@ -4,6 +4,8 @@ namespace vespalib { +VESPA_THREAD_STACK_TAG(unnamed_nonblocking_executor); + bool ThreadStackExecutor::acceptNewTask(MonitorGuard &) { @@ -15,8 +17,16 @@ ThreadStackExecutor::wakeup(MonitorGuard &) { } -ThreadStackExecutor::ThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit) : - ThreadStackExecutorBase(stackSize, taskLimit) +ThreadStackExecutor::ThreadStackExecutor(uint32_t threads, uint32_t stackSize, + uint32_t taskLimit) + : ThreadStackExecutorBase(stackSize, taskLimit, unnamed_nonblocking_executor) +{ + start(threads); +} + +ThreadStackExecutor::ThreadStackExecutor(uint32_t threads, uint32_t stackSize, + init_fun_t init_function, uint32_t taskLimit) + : ThreadStackExecutorBase(stackSize, taskLimit, std::move(init_function)) { start(threads); } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutor.h b/vespalib/src/vespa/vespalib/util/threadstackexecutor.h index 27d8c3da325..3ac081be05f 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutor.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutor.h @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once @@ -30,6 +29,12 @@ public: ThreadStackExecutor(uint32_t threads, uint32_t stackSize, uint32_t taskLimit = 0xffffffff); + // same as above, but enables you to specify a custom function + // used to wrap the main loop of all worker threads + ThreadStackExecutor(uint32_t threads, uint32_t stackSize, + init_fun_t init_function, + uint32_t taskLimit = 0xffffffff); + /** * Will invoke cleanup. **/ diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index 28f9a57b3c5..263ff528ebd 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -5,6 +5,12 @@ namespace vespalib { void +ThreadStackExecutorBase::ThreadInit::Run(FastOS_ThreadInterface *, void *) +{ + init_fun(worker); +} + +void ThreadStackExecutorBase::BlockedThread::wait() const { MonitorGuard guard(monitor); @@ -91,7 +97,7 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) } void -ThreadStackExecutorBase::Run(FastOS_ThreadInterface *, void *) +ThreadStackExecutorBase::run() { Worker worker; worker.verify(/* idle: */ true); @@ -107,7 +113,8 @@ ThreadStackExecutorBase::Run(FastOS_ThreadInterface *, void *) //----------------------------------------------------------------------------- ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t stackSize, - uint32_t taskLimit) + uint32_t taskLimit, + init_fun_t init_fun) : _pool(stackSize), _monitor(), _stats(), @@ -117,7 +124,8 @@ ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t stackSize, _barrier(), _taskCount(0), _taskLimit(taskLimit), - _closed(false) + _closed(false), + _thread_init(std::make_unique<ThreadInit>(*this, std::move(init_fun))) { assert(taskLimit > 0); } @@ -127,7 +135,7 @@ ThreadStackExecutorBase::start(uint32_t threads) { assert(threads > 0); for (uint32_t i = 0; i < threads; ++i) { - FastOS_ThreadInterface *thread = _pool.NewThread(this); + FastOS_ThreadInterface *thread = _pool.NewThread(_thread_init.get()); assert(thread != 0); (void)thread; } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index c2f5742009f..ee66ad71a38 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -7,17 +7,28 @@ #include "eventbarrier.hpp" #include "arrayqueue.hpp" #include "sync.h" +#include "runnable.h" #include <memory> #include <vector> #include <vespa/fastos/thread.h> namespace vespalib { +// Convenience macro used to create a function that can be used as an +// init function when creating an executor to inject a frame with the +// given name into the stack of all worker threads. + +#define VESPA_THREAD_STACK_TAG(name) \ + int name(Runnable &worker) { \ + worker.run(); \ + return 1; \ + } + /** * An executor service that executes tasks in multiple threads. **/ class ThreadStackExecutorBase : public ThreadExecutor, - public FastOS_Runnable + public Runnable { public: /** @@ -31,7 +42,17 @@ public: Stats() : maxPendingTasks(0), acceptedTasks(0), rejectedTasks(0) {} }; + using init_fun_t = std::function<int(Runnable&)>; + private: + struct ThreadInit : public FastOS_Runnable { + Runnable &worker; + init_fun_t init_fun; + explicit ThreadInit(Runnable &worker_in, init_fun_t init_fun_in) + : worker(worker_in), init_fun(std::move(init_fun_in)) {} + void Run(FastOS_ThreadInterface *, void *) override; + }; + struct TaggedTask { Task::UP task; uint32_t token; @@ -91,6 +112,7 @@ private: uint32_t _taskCount; uint32_t _taskLimit; bool _closed; + std::unique_ptr<ThreadInit> _thread_init; void block_thread(const LockGuard &, BlockedThread &blocked_thread); void unblock_threads(const MonitorGuard &); @@ -114,8 +136,8 @@ private: **/ bool obtainTask(Worker &worker); - // from FastOS_Runnable (all workers live here) - void Run(FastOS_ThreadInterface *, void *) override; + // Runnable (all workers live here) + void run() override; protected: /** @@ -155,8 +177,11 @@ protected: * * @param stackSize stack size per worker thread * @param taskLimit upper limit on accepted tasks + * @param init_fun custom function used to wrap the main loop of + * each worker thread. **/ - ThreadStackExecutorBase(uint32_t stackSize, uint32_t taskLimit); + ThreadStackExecutorBase(uint32_t stackSize, uint32_t taskLimit, + init_fun_t init_fun); /** * This will start the theads. This is to avoid starting tasks in diff --git a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp b/vespalib/src/vespa/vespalib/websocket/acceptor.cpp index d2217fb304d..830b82a8fa7 100644 --- a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp +++ b/vespalib/src/vespa/vespalib/websocket/acceptor.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include "acceptor.h" -#include <functional> #include <vespa/vespalib/net/socket_spec.h> +#include <functional> -namespace vespalib { -namespace ws { +namespace vespalib::ws { void Acceptor::accept_main(Handler<Socket> &socket_handler) @@ -34,4 +32,3 @@ Acceptor::~Acceptor() } } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/buffer.cpp b/vespalib/src/vespa/vespalib/websocket/buffer.cpp index de4c61bf9d9..b7c41b960a7 100644 --- a/vespalib/src/vespa/vespalib/websocket/buffer.cpp +++ b/vespalib/src/vespa/vespalib/websocket/buffer.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "buffer.h" -namespace vespalib { -namespace ws { +namespace vespalib::ws { void Buffer::ensure_free(size_t bytes) @@ -19,4 +16,3 @@ Buffer::ensure_free(size_t bytes) } } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/connection.cpp b/vespalib/src/vespa/vespalib/websocket/connection.cpp index d0a0af8dd33..cb304060271 100644 --- a/vespalib/src/vespa/vespalib/websocket/connection.cpp +++ b/vespalib/src/vespa/vespalib/websocket/connection.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "connection.h" +#include <cstdarg> +#include <cassert> -namespace vespalib { -namespace ws { +namespace vespalib::ws { namespace { @@ -205,4 +204,3 @@ Connection::flush() } } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/frame.cpp b/vespalib/src/vespa/vespalib/websocket/frame.cpp index 43aff81f599..76be06b1e1a 100644 --- a/vespalib/src/vespa/vespalib/websocket/frame.cpp +++ b/vespalib/src/vespa/vespalib/websocket/frame.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "frame.h" -namespace vespalib { -namespace ws { +namespace vespalib::ws { } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/handler.cpp b/vespalib/src/vespa/vespalib/websocket/handler.cpp index 28229e2816b..972154d0098 100644 --- a/vespalib/src/vespa/vespalib/websocket/handler.cpp +++ b/vespalib/src/vespa/vespalib/websocket/handler.cpp @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "handler.h" -namespace vespalib { -namespace ws { +namespace vespalib::ws { namespace { @@ -16,4 +13,3 @@ struct DummyItem {}; template class Handler<DummyItem>; } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/websocket/key.cpp b/vespalib/src/vespa/vespalib/websocket/key.cpp index 3091e924a48..12d8bd5cf4a 100644 --- a/vespalib/src/vespa/vespalib/websocket/key.cpp +++ b/vespalib/src/vespa/vespalib/websocket/key.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> #include "key.h" #include <vespa/vespalib/util/sha1.h> -namespace vespalib { -namespace ws { +namespace vespalib::ws { namespace { @@ -49,4 +46,3 @@ Key::accept(const vespalib::string &key) } } // namespace vespalib::ws -} // namespace vespalib diff --git a/vespalog/CMakeLists.txt b/vespalog/CMakeLists.txt index 17a91b8bbee..d613759e168 100644 --- a/vespalog/CMakeLists.txt +++ b/vespalog/CMakeLists.txt @@ -17,5 +17,5 @@ vespa_define_module( src/test/threads ) -vespa_install_script(src/logfmt/logfmt.pl logfmt bin) -install(FILES src/logfmt/logfmt.1 DESTINATION man/man1) +vespa_install_script(src/vespa-logfmt/vespa-logfmt.pl vespa-logfmt bin) +install(FILES src/vespa-logfmt/vespa-logfmt.1 DESTINATION man/man1) diff --git a/vespalog/src/logctl/.gitignore b/vespalog/src/logctl/.gitignore index 6089704ed9a..92957f69d15 100644 --- a/vespalog/src/logctl/.gitignore +++ b/vespalog/src/logctl/.gitignore @@ -1,3 +1,3 @@ .depend Makefile -logctl +vespa-logctl diff --git a/vespalog/src/logctl/CMakeLists.txt b/vespalog/src/logctl/CMakeLists.txt index 666e840fe60..23e7b6a10e8 100644 --- a/vespalog/src/logctl/CMakeLists.txt +++ b/vespalog/src/logctl/CMakeLists.txt @@ -1,8 +1,8 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalog_logctl_app +vespa_add_executable(vespalog_vespa-logctl_app SOURCES logctl.cpp - OUTPUT_NAME logctl + OUTPUT_NAME vespa-logctl INSTALL bin DEPENDS vespalog diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 8457fb944ce..2bae7c03589 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -1,20 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> -#include <cstdio> -#include <cstdlib> -#include <unistd.h> -#include <cstring> -#include <limits.h> - -#include <memory> -#include <vector> -#include <string> #include <vespa/defaults.h> #include <vespa/log/control-file.h> #include <vespa/log/internal.h> #include <vespa/log/component.h> -LOG_SETUP("logctl", "$Id$"); + +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +LOG_SETUP("vespa-logctl"); + using namespace ns_log; diff --git a/vespalog/src/logger/.gitignore b/vespalog/src/logger/.gitignore index 7cd5c615ee3..a7f5e36f079 100644 --- a/vespalog/src/logger/.gitignore +++ b/vespalog/src/logger/.gitignore @@ -1,5 +1,5 @@ .depend Makefile vespa-runserver -vlogconv -vlogreplay +vespa-log-conv +vespa-log-replay diff --git a/vespalog/src/logger/CMakeLists.txt b/vespalog/src/logger/CMakeLists.txt index 3e055e3b288..dba37539b09 100644 --- a/vespalog/src/logger/CMakeLists.txt +++ b/vespalog/src/logger/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(vespalog_vlogconv_app SOURCES logger.cpp llreader.cpp - OUTPUT_NAME vlogconv + OUTPUT_NAME vespa-log-conv INSTALL bin DEPENDS vespalog @@ -11,7 +11,7 @@ vespa_add_executable(vespalog_vlogconv_app vespa_add_executable(vespalog_vlogreplay_app SOURCES logreplay.c - OUTPUT_NAME vlogreplay + OUTPUT_NAME vespa-log-replay INSTALL bin DEPENDS m diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java index a3a69e8f4ed..5e75b10b302 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -155,6 +155,9 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { return ctrl; } synchronized(this) { + if (ctlFile == null) { + return defaultLevelCtrl; + } LevelController inherit = null; int lastdot = suffix.lastIndexOf('.'); diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp index 5b386d18e90..cb16b73fa24 100644 --- a/vespalog/src/test/bufferedlogskiptest.cpp +++ b/vespalog/src/test/bufferedlogskiptest.cpp @@ -1,18 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include <fstream> #include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <stdlib.h> #include <unistd.h> -#include <signal.h> -LOG_SETUP("bufferedlogskiptest", - "$Id$"); +LOG_SETUP("bufferedlogskiptest"); std::string readFile(const std::string& file) { std::ostringstream ost; diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp index 0e27bd65988..97f70ebe004 100644 --- a/vespalog/src/test/bufferedlogtest.cpp +++ b/vespalog/src/test/bufferedlogtest.cpp @@ -1,21 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger1.h" #include "bufferedlogtest.logger2.h" #include <fstream> #include <iostream> -#include <sstream> - -#include <sys/types.h> -#include <stdlib.h> #include <unistd.h> -#include <signal.h> -LOG_SETUP("bufferedlogtest", - "$Id$"); +LOG_SETUP("bufferedlogtest"); std::string readFile(const std::string& file) { std::ostringstream ost; diff --git a/vespalog/src/test/bufferedlogtest.logger1.cpp b/vespalog/src/test/bufferedlogtest.logger1.cpp index 992de19fea5..5af4b9489a4 100644 --- a/vespalog/src/test/bufferedlogtest.logger1.cpp +++ b/vespalog/src/test/bufferedlogtest.logger1.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger1.h" LOG_SETUP(".logger1"); diff --git a/vespalog/src/test/bufferedlogtest.logger2.cpp b/vespalog/src/test/bufferedlogtest.logger2.cpp index 688ecd153aa..ac6935c2c68 100644 --- a/vespalog/src/test/bufferedlogtest.logger2.cpp +++ b/vespalog/src/test/bufferedlogtest.logger2.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> #include "bufferedlogtest.logger2.h" LOG_SETUP(".logger2"); diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index a708777f350..90b8c13e497 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/app.h> #include <vespa/fastos/time.h> -#include <vespa/log/log.h> +#include <vespa/fastos/thread.h> +#include <vespa/log/bufferedlogger.h> #include <iostream> using std::string; - LOG_SETUP(".threadtest"); class FileThread : public FastOS_Runnable diff --git a/vespalog/src/logfmt/.gitignore b/vespalog/src/vespa-logfmt/.gitignore index 568d69eee3e..568d69eee3e 100644 --- a/vespalog/src/logfmt/.gitignore +++ b/vespalog/src/vespa-logfmt/.gitignore diff --git a/vespalog/src/logfmt/logfilter.c b/vespalog/src/vespa-logfmt/logfilter.c index 709ce2c016e..709ce2c016e 100644 --- a/vespalog/src/logfmt/logfilter.c +++ b/vespalog/src/vespa-logfmt/logfilter.c diff --git a/vespalog/src/logfmt/logfmt.1 b/vespalog/src/vespa-logfmt/vespa-logfmt.1 index a16b1ea2eb3..57c8d46370e 100644 --- a/vespalog/src/logfmt/logfmt.1 +++ b/vespalog/src/vespa-logfmt/vespa-logfmt.1 @@ -1,11 +1,11 @@ .\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $ .\" -.Dd October 29, 2004 -.Dt LOGFMT \&1 "Vespa documentation" -.Os "Yahoo! Vespa" "2.0" +.Dd Jun 7, 2017 +.Dt vespa-logfmt \&1 "Vespa documentation" +.Os "Yahoo! Vespa" "6.0" .Os .Sh NAME -.Nm logfmt +.Nm vespa-logfmt .Nd select and format messages from Vespa log files .Sh SYNOPSIS .Nm @@ -68,7 +68,7 @@ is shown. Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format. Note that the time is printed in the local timezone; to get GMT output use -.Nm "\*[q]env TZ=GMT logfmt\*[q]" +.Nm "\*[q]env TZ=GMT vespa-logfmt\*[q]" as your command. .It msecs Add milliseconds after the seconds in @@ -155,7 +155,7 @@ truncating if necessary. The command: .Pp .Bd -literal -offset indent -logfmt -l event -s service,message,fmttime,message +vespa-logfmt -l event -s service,message,fmttime,message .Ed .Pp will display only messages with log level "event", @@ -179,7 +179,7 @@ order is. The command: .Pp .Bd -literal -offset indent -logfmt -l all-info,-debug -s level \e +vespa-logfmt -l all-info,-debug -s level \e -s time,usecs,component,message -t -l -event .Ed .Pp diff --git a/vespalog/src/logfmt/logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl index 8d01feb3a6b..8d01feb3a6b 100755 --- a/vespalog/src/logfmt/logfmt.pl +++ b/vespalog/src/vespa-logfmt/vespa-logfmt.pl diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index 30f3dcb648d..db2021324c3 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -1,14 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/bufferedlogger.h> - -#include <iomanip> -#include <iostream> -#include <sstream> -#include <vector> -#include <stdarg.h> -#include <sys/time.h> - +#include "bufferedlogger.h" +#include <vespa/fastos/mutex.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> @@ -16,6 +9,13 @@ #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <iomanip> +#include <iostream> +#include <sstream> +#include <vector> +#include <cstdarg> +#include <ctime> + namespace ns_log { // implementation details for BufferedLogger diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h index 4f20f46af4c..9ab02f7842f 100644 --- a/vespalog/src/vespa/log/bufferedlogger.h +++ b/vespalog/src/vespa/log/bufferedlogger.h @@ -82,7 +82,6 @@ #define VESPA_LOG_LOGENTRYMAXAGE 300 // Max seconds an entry can be cached #define VESPA_LOG_COUNTAGEFACTOR 5 // How many seconds each count counts for -#include <vespa/fastos/mutex.h> #include <vespa/log/log.h> #include <sstream> #include <string> @@ -199,7 +198,6 @@ public: /** Trim the buffer. Removing old messages if wanted. */ void trimCache(); - }; } // ns_log diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp index 07b638d7f84..e88f1018675 100644 --- a/vespalog/src/vespa/log/log-target-file.cpp +++ b/vespalog/src/vespa/log/log-target-file.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> #include <unistd.h> #include <cstring> -#include <sys/stat.h> #include <fcntl.h> -#include <cstdlib> -#include <errno.h> -#include <cstdio> +#include <cerrno> +#include <cassert> #include "log.h" LOG_SETUP(".log"); -LOG_RCSID("$Id$"); #include "log-target-file.h" #include "internal.h" diff --git a/vespalog/src/vespa/log/log-target.cpp b/vespalog/src/vespa/log/log-target.cpp index d34e0067b04..b6692caa308 100644 --- a/vespalog/src/vespa/log/log-target.cpp +++ b/vespalog/src/vespa/log/log-target.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <sys/types.h> #include <cstring> -#include <cstdlib> +#include <cassert> #include "log.h" LOG_SETUP(".log"); -LOG_RCSID("$Id$"); #include "log-target-fd.h" #include "log-target-file.h" diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp index 7e92008504e..c31d110d241 100644 --- a/vespalog/src/vespa/log/log.cpp +++ b/vespalog/src/vespa/log/log.cpp @@ -10,6 +10,7 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #include "log-target.h" #include "internal.h" #include "control-file.h" +#include "bufferedlogger.h" #include <vespa/fastos/thread.h> @@ -222,6 +223,7 @@ Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...) actualSize = tryLog(sizeofPayload, level, file, line, fmt, args); va_end(args); } while (sizeofPayload < actualSize); + ns_log::BufferedLogger::logger.trimCache(); } void Logger::doLogCore(uint64_t timestamp, LogLevel level, diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index a14ff87e1be..53a0eec140c 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -2,7 +2,7 @@ #pragma once #include <memory> -#include <inttypes.h> +#include <cstdint> #include <sys/types.h> // for pid_t #include <new> // for placement new #include <cstdlib> // for malloc @@ -38,14 +38,12 @@ static int log_dummmy __attribute__((unused)) = logger.setRcsId(x) do { \ if (logger.wants(ns_log::Logger::level)) { \ logger.doLog(ns_log::Logger::level, __FILE__, __LINE__, __VA_ARGS__); \ - ns_log::BufferedLogger::logger.trimCache(); \ } \ } while (0) #define VLOG(level, ...) \ do { \ if (logger.wants(level)) { \ logger.doLog(level, __FILE__, __LINE__, __VA_ARGS__); \ - ns_log::BufferedLogger::logger.trimCache(); \ } \ } while (0) #endif @@ -302,6 +300,3 @@ extern void log_abort(const char *message, #else #define LOG_ASSERT(expr) #endif // #ifndef NDEBUG - -#include <vespa/log/bufferedlogger.h> - diff --git a/vsm/src/tests/create-test.sh b/vsm/src/tests/create-test.sh index c4259526089..63dafe7e3ce 100755..100644 --- a/vsm/src/tests/create-test.sh +++ b/vsm/src/tests/create-test.sh @@ -24,7 +24,6 @@ gen_source() { echo "generating '$1' ..." echo "#include <vespa/log/log.h>" > $1 echo "LOG_SETUP(\"${test}_test\");" >> $1 - echo "#include <vespa/fastos/fastos.h>" >> $1 echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1 echo "" >> $1 echo "// using namespace ;" >> $1 diff --git a/vsm/src/tests/docsum/docsum.cpp b/vsm/src/tests/docsum/docsum.cpp index d854421ecb5..1b0fbf70623 100644 --- a/vsm/src/tests/docsum/docsum.cpp +++ b/vsm/src/tests/docsum/docsum.cpp @@ -1,8 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> - -#include <vector> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/structdatatype.h> #include <vespa/vsm/common/docsum.h> #include <vespa/vsm/vsm/flattendocsumwriter.h> #include <vespa/vsm/vsm/slimefieldwriter.h> diff --git a/vsm/src/tests/document/document.cpp b/vsm/src/tests/document/document.cpp index 35ca55826ba..629e2954953 100644 --- a/vsm/src/tests/document/document.cpp +++ b/vsm/src/tests/document/document.cpp @@ -2,10 +2,10 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vsm/common/storagedocument.h> #include <vespa/vespalib/stllike/asciistream.h> - using namespace document; namespace vsm { diff --git a/vsm/src/vespa/vsm/common/document.h b/vsm/src/vespa/vsm/common/document.h index a0341aaf7c0..ea0f9216ab9 100644 --- a/vsm/src/vespa/vsm/common/document.h +++ b/vsm/src/vespa/vsm/common/document.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/query/base.h> #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <map> namespace vespalib { class asciistream; diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/vsm/src/vespa/vsm/common/documenttypemapping.cpp index 96a7effae57..9a8bbb28f65 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp +++ b/vsm/src/vespa/vsm/common/documenttypemapping.cpp @@ -2,6 +2,7 @@ #include "documenttypemapping.h" #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/vsm/src/vespa/vsm/searcher/fieldsearcher.h index 3b82f68072b..e28275536f3 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h +++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/query/query.h> #include <vespa/vsm/common/document.h> #include <vespa/vsm/common/storagedocument.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace vsm { @@ -79,9 +80,8 @@ public: size_t maxFieldLength() const { return _maxFieldLength; } private: - class IteratorHandler : public document::FieldValue::IteratorHandler { + class IteratorHandler : public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; FieldSearcher & _searcher; void onPrimitive(uint32_t fid, const Content & c) override; diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp index 8e444102930..c713e0284ac 100644 --- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "utf8stringfieldsearcherbase.h" +#include <cassert> using search::QueryTerm; using search::QueryTermList; diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp index 23e64d0bf5e..b5d49f463ef 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "utf8substringsnippetmodifier.h" +#include <cassert> using search::byte; using search::QueryTerm; diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index 961add6d7d6..b7ed6c0b7da 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -4,6 +4,7 @@ #include "slimefieldwriter.h" #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.docsumfilter"); @@ -13,9 +14,8 @@ using namespace search::docsummary; namespace { -class Handler : public document::FieldValue::IteratorHandler { +class Handler : public document::fieldvalue::IteratorHandler { public: - typedef document::FieldValue::IteratorHandler::Content Content; }; struct IntResultHandler : public Handler { diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h index 9c6ba8dae9c..aa58a6ae6df 100644 --- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h +++ b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/vsm/common/charbuffer.h> namespace vsm { @@ -10,10 +11,8 @@ namespace vsm { * This class is used to flatten out and write a complex field value. * A separator string is inserted between primitive field values. **/ -class FlattenDocsumWriter : public document::FieldValue::IteratorHandler { +class FlattenDocsumWriter : public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; - CharBuffer _output; vespalib::string _separator; bool _useSeparator; diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.h b/vsm/src/vespa/vsm/vsm/snippetmodifier.h index 17d6bb44d4c..182f09d5d55 100644 --- a/vsm/src/vespa/vsm/vsm/snippetmodifier.h +++ b/vsm/src/vespa/vsm/vsm/snippetmodifier.h @@ -7,6 +7,7 @@ #include <vespa/vsm/common/fieldmodifier.h> #include <vespa/vsm/searcher/utf8substringsnippetmodifier.h> #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/document/fieldvalue/iteratorhandler.h> namespace vsm { @@ -19,11 +20,9 @@ namespace vsm { * responsible for modifying the field value by inserting unit separators before and after matches. * A group separator is inserted between primitive field values the same way as done by FlattenDocsumWriter. **/ -class SnippetModifier : public FieldModifier, public document::FieldValue::IteratorHandler +class SnippetModifier : public FieldModifier, public document::fieldvalue::IteratorHandler { private: - typedef document::FieldValue::IteratorHandler::Content Content; - UTF8SubstringSnippetModifier::SP _searcher; CharBuffer::SP _valueBuf; // buffer to store the final modified field value char _groupSep; |