diff options
675 files changed, 5904 insertions, 11228 deletions
diff --git a/.gitignore b/.gitignore index c9bd860bf2f..8f47a77af57 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ CMakeCache.txt CTestTestfile.cmake cmake_install.cmake Makefile +Testing diff --git a/README-documentation.md b/README-documentation.md new file mode 100644 index 00000000000..0cc8db75cfb --- /dev/null +++ b/README-documentation.md @@ -0,0 +1,80 @@ +# Creating Vespa documentation + +All features of Vespa must have user documentation. +This document explains how to add documentation. + +## Practical information + +Vespa documentation is served using straightforward +[GitHub Project pages](https://help.github.com/categories/github-pages-basics/) +with +[Jekyll](https://help.github.com/categories/customizing-github-pages/). +To edit documentation check out and work off the branch gh-pages from the +[documentation repository for now](https://git.corp.yahoo.com/vespa/documentation). +This branch contains documentation instead of the code. + +Documentation are written in straightforward HTML. +We use a single Jekyll template (_layouts/default.html) to add header, footer and layout. +With Jekyll installed (follow the link above) you can use + + bundle exec jekyll serve + +to set up a local server at localhost:4000 to see the pages as they will look when served. + +The layout is written in Bootstrap and we refer directly to the Bootstrap CSS. +Refer to [Bootstrap documentation](http://getbootstrap.com/css/) if you need to +add style effects to your page. Note that the entire documentation page content +is contained in a Bootstrap layout column with column width 12. Please do not add custom style sheets +as it is a pain to maintain. + +## Writing good documentation + +Please read the following guide to writing good documentation before writing some. + +### Guides and references + +A document cannot be both comprehensive and comprehensible. +Because of this we split our documentation into *guides* and *reference* documents. + +Guides should be easy to understand by only explaining the most important concepts under discussion. +Reference documents on the other hand must be complete but should skip verbiage meant to aid understanding. + +Reference documents are those that are placed in reference/ subdirectories +(with the exception of parts of content/, where this separation became messed up at some point). + +### Maintainability + +Prioritize maintainability higher than usability: + +* Don't include unnecessary details, especially ephemeral ones such as that a feature is "recently added" or how things was before, etc. The guide/reference distinction helps here: Guides are harder to maintain as they contain more verbiage, and they should not unnecessarily repeat information found in a reference doc. **Write such that the document will still be correct in a half decade.** + +* Don't repeat information found in other documents. It is tempting to make life easier for users by writing use-case oriented documentation on how to accomplish specific tasks, but this backfires as it leads to a lot of repetition which we fail to maintain. In the long run it is better to explain the concepts clearly and succinctly and leave it to the users to piece together the information. **Use the same principles for documentation as for code: DRY, refactor for coherency etc.** + +* Be wary of adding code in the documentation. The code will becomes incorrect over time and should in most cases be placed in git as continuously built code and referenced from the doc. + +### Text quality + +Documentation is not high prose, and not a podcast. +Users want to consume the information as soon as possible with as little effort as possible and get on with their lives. + +Make the text as short, clear, and easy to read as possible: +* Describe things plainly "as they are". You usually shouldn't worry about explaining why, what you can do with it etc. +* Use short sentences with simple structure. +* Avoid superfluous words such as "very". +* Avoid filler sentences intended to improve the flow of the text - documents are usually browsed, not read anyway. +* Use consistent terminology even when it leads to repetition which would be bad in other kinds of writing. +* Use active form "index the documents", not passive "indexing the documents". +* Avoid making it personal - do not use "we", "you", "our". +* Do not use &quot; , &mdash; and the likes - makes the document harder to edit, and no need to use it. +* Less is more - <em> and <strong> is sufficient formatting in most cases. + +### Links + +Add an *id* attribute to each heading such that link can refer to it: Use the exact same text as the heading as id, lowercased and with spaces replaced by dashes such that references +can be made without checking the source. Don't change headings/ids unless completely necessary as that breaks links. + +Example: +<h2 id="my-nice-heading">My nice Heading</h2> +If this algorithmic transformation is followed it is possible to link to this section using <a href="doc.html#my-nice-heading"> without having to consult the html source of the page to find the right id. + +*By Jon Bratseth in June 2016* diff --git a/README.md b/README.md index 3004fd323c4..c1ab45629f0 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,34 @@ Vespa is an engine for low-latency computation over large data sets. It stores and indexes your data such that queries, selection and processing over the data can be performed at serving time. -This README describes how to build and develop the Vespa engine. -For user documentation see TODO: Github pages link +This README describes how to build and develop the Vespa engine. If you want to use Vespa +you can go to the +[quick start guide](http://yahoo.github.io/vespa/vespa-quick-start.html), or find the full +documentation and other resources at http://yahoo.github.io/vespa/. -## Getting started +## Getting started developing ### Setting up local git config git config --global user.name "John Doe" git config --global user.email johndoe@host.com - ### Setting up build environment - sudo yum -y install epel-release - # TODO: Install build deps or depend on Build-Require in .spec file? +C++ building is supported on CentOS 7. + +#### Install required build dependencies + + sudo yum -y install epel-release centos-release-scl yum-utils + sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo + sudo yum -y install devtoolset-4-gcc-c++ devtoolset-4-libatomic-devel \ + Judy-devel cmake3 ccache lz4-devel zlib-devel maven libicu-devel llvm-devel \ + llvm-static java-1.8.0-openjdk-devel openssl-devel rpm-build make \ + vespa-boost-devel vespa-libtorrent-devel vespa-zookeeper-c-client-devel vespa-cppunit-devel + +or use our prebuilt docker image + + # TODO: Add docker command ### Building Java modules @@ -29,11 +42,8 @@ Java modules can be built on any environment having Java and Maven: ### Building C++ modules -C++ building is currently supported on CentOS 7: - -TODO: List required build dependencies - - sh bootstrap.sh + source /opt/rh/devtoolset-4/enable + sh bootstrap.sh full cmake . make make test @@ -47,5 +57,8 @@ TODO: List required build dependencies * OS X : See [node-admin/README_MAC.md](node-admin/README_MAC.md) * Linux : See [node-admin/README_LINUX.md](node-admin/README_LINUX.md) - Code licensed under the Apache 2.0 license. See LICENSE file for terms. + +## Documenting your features + +See [README-documentation.md](README-documentation.md). diff --git a/annotations/pom.xml b/annotations/pom.xml index 59dd7e82ac9..84378d2aa66 100644 --- a/annotations/pom.xml +++ b/annotations/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>annotations</artifactId> <packaging>jar</packaging> diff --git a/application-deploy-plugin/pom.xml b/application-deploy-plugin/pom.xml index 3a8d785b749..fbed02aa934 100644 --- a/application-deploy-plugin/pom.xml +++ b/application-deploy-plugin/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-deploy-plugin</artifactId> <version>6-SNAPSHOT</version> diff --git a/application-model/pom.xml b/application-model/pom.xml index f03a71032b2..388d20ea885 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-model</artifactId> <packaging>container-plugin</packaging> diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index dc624bfed4d..1fed278feb1 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-preprocessor</artifactId> <packaging>jar</packaging> diff --git a/application-preprocessor/src/main/sh/preprocess-application b/application-preprocessor/src/main/sh/vespa-preprocess-application index 68c7fa23d00..68c7fa23d00 100644 --- a/application-preprocessor/src/main/sh/preprocess-application +++ b/application-preprocessor/src/main/sh/vespa-preprocess-application diff --git a/application/pom.xml b/application/pom.xml index 3132108a378..ff6685cec73 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application</artifactId> <packaging>jar</packaging> diff --git a/bootstrap.sh b/bootstrap.sh index f41f3369b32..940179ef020 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,6 +1,34 @@ #!/bin/bash -e # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +usage() { + echo "Usage: $0 [full | java | default]" >&2 +} + +if [ $# -eq 0 ]; then + # Build minimal set of java modules required to run cmake + MODE=default +elif [ "$1" = "full" ]; then + # Build all java modules required by C++ testing + MODE=full +elif [ "$1" = "java" ]; then + # Build minial set of java modules requires to run mvn install from the source root + MODE=java +elif [ "$1" = "default" ]; then + : +elif [ "$1" = "-h" -o "$1" = "--help" ]; then + usage + exit 0 +else + echo "Unknown argument: $1" >&2 + usage + exit 1 +fi + +mvn_install() { + mvn install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true $@ +} + # Generate vtag map top=$(dirname $0) $top/dist/getversion.pl -M $top > $top/dist/vtag.map @@ -8,19 +36,25 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map # These modules are required to be available to maven for it to calculate the # Vespa java dependency graph. MODULES=" - parent - configgen + . annotations scalalib bundle-plugin - config-class-plugin - yolean - vespajlib - jrt - component - messagebus - filedistributionmanager" + " for module in $MODULES; do - (cd $module && mvn install -DskipTests -Dmaven.javadoc.skip=true) + (cd $module && mvn_install -N) done + +mvn_install -am -pl config-class-plugin -rf configgen + +case "$MODE" in + java) + ;; + full) + mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus -rf yolean + ;; + default) + mvn_install -am -pl filedistributionmanager -rf yolean + ;; +esac diff --git a/bundle-plugin-test/pom.xml b/bundle-plugin-test/pom.xml index 0ca63d82fec..a4850c73dbb 100644 --- a/bundle-plugin-test/pom.xml +++ b/bundle-plugin-test/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.container.maven.plugin</groupId> <artifactId>bundle-plugin-test</artifactId> diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index a57d1a41bf8..604d0f1e6c7 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>bundle-plugin</artifactId> <version>6-SNAPSHOT</version> diff --git a/chain/pom.xml b/chain/pom.xml index fe83d2bfe74..9d65d9fd2bb 100755 --- a/chain/pom.xml +++ b/chain/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>chain</artifactId> <packaging>jar</packaging> diff --git a/clustercontroller-apps/OWNERS b/clustercontroller-apps/OWNERS index b3db17e22d8..de6ac0dd8f5 100644 --- a/clustercontroller-apps/OWNERS +++ b/clustercontroller-apps/OWNERS @@ -1,2 +1,2 @@ vekterli -hakon +hakonhall diff --git a/clustercontroller-apps/pom.xml b/clustercontroller-apps/pom.xml index b8103cd0210..8830c00f89d 100644 --- a/clustercontroller-apps/pom.xml +++ b/clustercontroller-apps/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-apps</artifactId> <version>6-SNAPSHOT</version> diff --git a/clustercontroller-apputil/OWNERS b/clustercontroller-apputil/OWNERS index b3db17e22d8..de6ac0dd8f5 100644 --- a/clustercontroller-apputil/OWNERS +++ b/clustercontroller-apputil/OWNERS @@ -1,2 +1,2 @@ vekterli -hakon +hakonhall diff --git a/clustercontroller-apputil/pom.xml b/clustercontroller-apputil/pom.xml index 261757bbf6f..5dc2c2a83d9 100644 --- a/clustercontroller-apputil/pom.xml +++ b/clustercontroller-apputil/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-apputil</artifactId> <version>6-SNAPSHOT</version> diff --git a/clustercontroller-core/OWNERS b/clustercontroller-core/OWNERS index b5f3398d1a0..abe8e49c8d6 100644 --- a/clustercontroller-core/OWNERS +++ b/clustercontroller-core/OWNERS @@ -1,3 +1,3 @@ vekterli -hakon +hakonhall bratseth diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index a010d88915b..93317a54dec 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-core</artifactId> <version>6-SNAPSHOT</version> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java index 19f8f81c628..4274b8098d1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java @@ -17,7 +17,7 @@ import java.util.TreeMap; /** * Detail information about the current state of all the distributor and storage nodes of the cluster. * - * @author hakon + * @author hakonhall * @author bratseth */ public class ClusterInfo { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java index 3444f4c2540..328acfb4dbe 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java @@ -22,7 +22,7 @@ import java.util.logging.Logger; * are mostly represented by the ClusterState. The dynamic parts include stats for tracking outstanding * merges before steady-state is reached. * - * @author hakon + * @author hakonhall * @since 5.33 */ public class ClusterStateView { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java index a52034d10a2..4c0b582c3d3 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregator.java @@ -29,7 +29,7 @@ import com.yahoo.log.LogLevel; * for a given storage nodes. So we need to keep track of the latest info * from each distributor. * - * @author hakon + * @author hakonhall * @since 5.34 */ public class ClusterStatsAggregator { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java index 32c68aff083..1884dee5863 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNodeStatsBridge; * Class encapsulating what the Cluster Controller knows about a distributor node. Most of the information is * common between Storage- and Distributor- nodes, and stored in the base class NodeInfo. * - * @author hakon + * @author hakonhall */ public class DistributorNodeInfo extends NodeInfo { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/LatencyStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/LatencyStats.java index 482bfbf6004..6cdbb50e1eb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/LatencyStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/LatencyStats.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; /** * LatencyStats handles adding latencies and counts. - * @author hakon + * @author hakonhall */ public class LatencyStats { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index ce6686c3ad9..433df4c2249 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -81,11 +81,11 @@ public class MetricUpdater { } public void recordNewNodeEvent() { - // TODO(hakon): Replace add() with a persistent aggregate metric. + // TODO(hakonhall): Replace add() with a persistent aggregate metric. metricReporter.add("node-event", 1); } public void updateMergeOpMetrics(Map<String, NodeMergeStats> storageNodeStats) { - // TODO(hakon): Remove this method once we figure out how to propagate metrics to state HTTP API. + // TODO(hakonhall): Remove this method once we figure out how to propagate metrics to state HTTP API. } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java index 67f11574c5b..22d9fc0cac8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeMergeStats.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNode; /** - * @author hakon + * @author hakonhall * @since 5.33 */ public class NodeMergeStats { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java index 12018ac4b6f..df0d2f78c61 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java @@ -14,7 +14,7 @@ import java.util.List; /** * Checks if a node can be upgraded. * - * @author dybdahl + * @author dybis */ public class NodeStateChangeChecker { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodes.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodes.java index a7e34c7321f..186d78f35e4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodes.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodes.java @@ -8,7 +8,7 @@ import java.util.Map; * by Distributors from their getnodestate RPCs. The stats for a single storage node * is represented by the StorageNodeStats class. * - * @author hakon + * @author hakonhall */ public class StatsForStorageNodes { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java index 2c719f9fb3c..40f704922fe 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageMergeStats.java @@ -9,7 +9,7 @@ import java.util.Set; /** * Class for storing the pending merge operation stats for all the storage nodes. * - * @author hakon + * @author hakonhall * @since 5.34 */ public class StorageMergeStats implements Iterable<NodeMergeStats> { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeInfo.java index 86f8be36a9d..1cafdd1ed0f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeInfo.java @@ -9,7 +9,7 @@ import com.yahoo.vdslib.state.NodeType; * Class encapsulating what the Cluster Controller knows about a storage node. Most of the information is * common between Storage- and Distributor- nodes, and stored in the base class NodeInfo. * - * @author hakon + * @author hakonhall */ public class StorageNodeInfo extends NodeInfo { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStats.java index c46e489453b..5b87b86bc0f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStats.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStats.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; /** * Contains stats related to a single storage node. * - * @author hakon + * @author hakonhall */ public class StorageNodeStats { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainer.java index bb01bf80d77..ca8fcfb97ea 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainer.java @@ -9,7 +9,7 @@ import java.util.Map; * by Distributors from their getnodestate RPCs. The stats for a single storage node * is represented by the StorageNodeStats class. * - * @author hakon + * @author hakonhall */ public class StorageNodeStatsContainer { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 69c5e10246c..21c4a7f677b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -84,7 +84,7 @@ public class MasterDataGatherer { private class DirCallback implements AsyncCallback.ChildrenCallback { public void processResult(int version, String path, Object context, List<String> nodes) { if (nodes == null) nodes = new LinkedList<String>(); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); + log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); // Detect what nodes are added and what nodes have been removed. Others can be ignored. List<Integer> addedNodes = new LinkedList<Integer>(); synchronized (nextMasterData) { @@ -103,7 +103,7 @@ public class MasterDataGatherer { nextMasterData.remove(index); } for (Integer index : addedNodes) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + zooKeeperRoot + index + "' to see vote"); + log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + zooKeeperRoot + index + "' to see vote"); nextMasterData.put(index, null); session.getData(zooKeeperRoot + "indexes/" + index, changeWatcher, nodeListener, null); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Distributor.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Distributor.java index c48c203a55c..28bb10c4a98 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Distributor.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Distributor.java @@ -8,7 +8,7 @@ import java.util.List; /** * Class for handling Distributor part of HostInfo. - * @author dybdahl + * @author dybis */ public class Distributor { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java index fbc53128415..fd7c4db30b8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java @@ -11,7 +11,7 @@ import java.util.logging.Logger; /** * Parsing and keeping of host info from nodes. - * @author dybdahl + * @author dybis */ public class HostInfo { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java index 23c502063a6..b2924516f26 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Metrics.java @@ -9,7 +9,7 @@ import java.util.List; /** * Keeper for Metrics for HostInfo. - * @author dybdahl + * @author dybis */ public class Metrics { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java index 4a8cff2d5bb..c208c0a3e19 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNode.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * Keeping information about a storage node seen from the distributor. - * @author dybdahl + * @author dybis */ public class StorageNode { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java index ed4664c5b44..980a8b4681f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java @@ -13,7 +13,7 @@ import java.util.Map; /** * Class used to create a StorageNodeStatsContainer from HostInfo. - * @author hakon + * @author hakonhall */ public class StorageNodeStatsBridge { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Vtag.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Vtag.java index b4d445c1844..c18be1cea64 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Vtag.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/Vtag.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * Class for handling version. - * @author dybdahl + * @author dybis */ public class Vtag { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java index a62a8676096..8583f57935b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java @@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; /** - * @author hakon + * @author hakonhall * @since 5.34 */ public class ClusterStateViewTest { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java index e87cad135c8..eab5f3ac0da 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java @@ -14,7 +14,7 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; /** - * @author hakon + * @author hakonhall * @since 5.34 */ @RunWith(MockitoJUnitRunner.class) diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodeTest.java index ddf0286b0fe..8cb62beda15 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatsForStorageNodeTest.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; /** - * @author hakon + * @author hakonhall */ public class StatsForStorageNodeTest { @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainerTest.java index e2832c5b6b9..958218c3585 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsContainerTest.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; /** - * @author hakon + * @author hakonhall */ public class StorageNodeStatsContainerTest { @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsTest.java index b905cd32979..d75fabbbb94 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StorageNodeStatsTest.java @@ -6,7 +6,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; /** - * @author hakon + * @author hakonhall */ public class StorageNodeStatsTest { @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java index 9d23031cd55..77e2c771f41 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridgeTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; /** - * @author hakon + * @author hakonhall */ public class StorageNodeStatsBridgeTest { diff --git a/clustercontroller-standalone/OWNERS b/clustercontroller-standalone/OWNERS index b3db17e22d8..de6ac0dd8f5 100644 --- a/clustercontroller-standalone/OWNERS +++ b/clustercontroller-standalone/OWNERS @@ -1,2 +1,2 @@ vekterli -hakon +hakonhall diff --git a/clustercontroller-standalone/pom.xml b/clustercontroller-standalone/pom.xml index 0569a8976f5..b9ddc4c9396 100644 --- a/clustercontroller-standalone/pom.xml +++ b/clustercontroller-standalone/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-standalone</artifactId> <version>6-SNAPSHOT</version> diff --git a/clustercontroller-utils/OWNERS b/clustercontroller-utils/OWNERS index b3db17e22d8..de6ac0dd8f5 100644 --- a/clustercontroller-utils/OWNERS +++ b/clustercontroller-utils/OWNERS @@ -1,2 +1,2 @@ vekterli -hakon +hakonhall diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index 10d6c220d86..7cf7cf0fdde 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-utils</artifactId> <version>6-SNAPSHOT</version> diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java index 4a0cb76f278..8b418d464d8 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/errors/MissingResourceException.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.errors; /** - * @author hakon + * @author hakonhall */ public class MissingResourceException extends StateRestApiException { public MissingResourceException(String resource) { diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java index 9f4ecac84d4..38518a63d99 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/response/SetResponse.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.response; /** * The response of a set operation. - * @author dybdahl + * @author dybis */ public class SetResponse { private final String reason; diff --git a/component/pom.xml b/component/pom.xml index 038e154f20f..15ed4a190f2 100755 --- a/component/pom.xml +++ b/component/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>component</artifactId> <packaging>container-plugin</packaging> diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index f2a66275c70..7f49e9fb8b6 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-application-package</artifactId> <packaging>container-plugin</packaging> @@ -93,11 +92,6 @@ <artifactId>jackson-databind</artifactId> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -122,7 +116,6 @@ <configuration> <compilerArgs> <arg>-Xlint:all</arg> - <arg>-Xlint:-try</arg> <arg>-Werror</arg> </compilerArgs> </configuration> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java index 8f2026afc66..b7e80a6abe8 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java @@ -1,16 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.application.provider; -import com.google.common.base.Charsets; import com.yahoo.collections.Tuple2; -import com.yahoo.config.codegen.CNode; import com.yahoo.vespa.config.util.ConfigUtils; import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.ZipEntry; import java.util.zip.ZipException; @@ -171,7 +169,7 @@ public class Bundle { return new StringReader(""); } try { - return new InputStreamReader(bundle.jarFile.getInputStream(zipEntry), Charsets.UTF_8); + return new InputStreamReader(bundle.jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8); } catch (IOException e) { throw new IllegalArgumentException("IOException", e); } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/ConfigDefinitionDirTest.java b/config-application-package/src/test/java/com/yahoo/config/application/ConfigDefinitionDirTest.java index ada517c9a5c..5d0766229e4 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/ConfigDefinitionDirTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/ConfigDefinitionDirTest.java @@ -1,10 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import com.google.common.io.Files; import com.yahoo.config.model.application.provider.Bundle; import com.yahoo.io.IOUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -23,9 +24,12 @@ public class ConfigDefinitionDirTest { private static final String bundleFileName = "com.yahoo.searcher1.jar"; private static final File bundleFile = new File("src/test/resources/defdircomponent/" + bundleFileName); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test public void require_that_defs_are_added() throws IOException { - File defDir = Files.createTempDir(); + File defDir = temporaryFolder.newFolder(); ConfigDefinitionDir dir = new ConfigDefinitionDir(defDir); Bundle bundle = new Bundle(new JarFile(bundleFile), bundleFile); assertThat(defDir.listFiles().length, is(0)); @@ -36,7 +40,7 @@ public class ConfigDefinitionDirTest { @Test public void require_that_conflicting_defs_are_not_added() throws IOException { - File defDir = Files.createTempDir(); + File defDir = temporaryFolder.newFolder(); IOUtils.writeFile(new File(defDir, "foo.def"), "alreadyexists", false); ConfigDefinitionDir dir = new ConfigDefinitionDir(defDir); Bundle bundle = new Bundle(new JarFile(bundleFile), bundleFile); diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java index ac207de7231..b1c0d78b299 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java @@ -1,10 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.application.provider; -import com.google.common.io.Files; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationFileTest; import com.yahoo.path.Path; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import java.io.File; @@ -14,9 +15,12 @@ import java.io.File; */ public class FilesApplicationFileTest extends ApplicationFileTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Override public ApplicationFile getApplicationFile(Path path) throws Exception { - File tmp = Files.createTempDir(); + File tmp = temporaryFolder.newFolder(); writeAppTo(tmp); return new FilesApplicationFile(path, new File(tmp, path.getRelative())); } diff --git a/config-bundle/pom.xml b/config-bundle/pom.xml index d439e7e261b..dac015ea305 100644 --- a/config-bundle/pom.xml +++ b/config-bundle/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-bundle</artifactId> <packaging>container-plugin</packaging> diff --git a/config-class-plugin/pom.xml b/config-class-plugin/pom.xml index ce37260819b..9dbcb5c86bb 100644 --- a/config-class-plugin/pom.xml +++ b/config-class-plugin/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-class-plugin</artifactId> <packaging>maven-plugin</packaging> diff --git a/config-lib/pom.xml b/config-lib/pom.xml index 80568a1ea26..c5c0bc2e2c9 100644 --- a/config-lib/pom.xml +++ b/config-lib/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-lib</artifactId> <packaging>container-plugin</packaging> diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 7280ac224ee..cf016ec4ee0 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-api</artifactId> <version>6-SNAPSHOT</version> diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 0864a502783..da187e839c8 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-fat</artifactId> <packaging>bundle</packaging> diff --git a/config-model/pom.xml b/config-model/pom.xml index 565ae7ab09f..e526b12c10c 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model</artifactId> <packaging>container-plugin</packaging> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java index 6082ca9f72d..c998307f3f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java @@ -70,6 +70,7 @@ public class DefaultMetricConsumers { // transaction log metrics.add(new Metric("content.proton.transactionlog.entries.average")); + metrics.add(new Metric("content.proton.transactionlog.disk_usage.average")); // document store metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average")); @@ -154,7 +155,7 @@ public class DefaultMetricConsumers { metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count")); metrics.add(new Metric("cluster-controller.is-master.last")); - // TODO(hakon): Update this name once persistent "count" metrics has been implemented. + // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java index 4b0e350718c..07657925d03 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -14,7 +14,7 @@ import java.util.stream.Collectors; /** * Validates the changes between a current and next document type used in a document database. * - * @author <a href="mailto:Tor.Egge@yahoo-inc.com">Tor Egge</a> + * @author toregge * @since 2014-11-25 */ public class DocumentTypeChangeValidator { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index fa5d46526f4..c8d2e868e42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -70,6 +70,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui handleAttribute(e, t.searchNode); } else if (equals("summary", e)) { handleSummary(e, t.searchNode); + } else if (equals("initialize", e)) { + handleInitialize(e, t.searchNode); } } } @@ -244,4 +246,14 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } } + + private void handleInitialize(Element spec, Tuning.SearchNode sn) { + sn.initialize = new Tuning.SearchNode.Initialize(); + for (Element e : XML.getChildren(spec)) { + if (equals("threads", e)) { + sn.initialize.threads = asInt(e); + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index e09d47e7bc1..24cb1a16d08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -73,7 +73,6 @@ public class BundleMapper { bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.KeyValueSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.QuerySnapshotSearcher", searchAndDocprocBundle); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 25a98353f74..1841f475e76 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -167,7 +167,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } } - public static class Attribute implements ProtonConfig.Producer { + public static class Attribute implements ProtonConfig.Producer { public static class Io implements ProtonConfig.Producer { public IoType write = null; @@ -337,12 +337,24 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } } + public static class Initialize implements ProtonConfig.Producer { + public Integer threads = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (threads != null) { + builder.initialize(new ProtonConfig.Initialize.Builder().threads(threads)); + } + } + } + public RequestThreads threads = null; public FlushStrategy strategy = null; public Resizing resizing = null; public Index index = null; public Attribute attribute = null; public Summary summary = null; + public Initialize initialize = null; @Override public void getConfig(ProtonConfig.Builder builder) { @@ -352,6 +364,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P if (index != null) index.getConfig(builder); if (attribute != null) attribute.getConfig(builder); if (summary != null) summary.getConfig(builder); + if (initialize != null) initialize.getConfig(builder); } } diff --git a/config-model/src/main/perl/deploy b/config-model/src/main/perl/deploy index 00e3b0d91f8..31dda77b99e 100755 --- a/config-model/src/main/perl/deploy +++ b/config-model/src/main/perl/deploy @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # This script is for uploading, preparing, activating and fetching # application packages to a cloud config server @@ -6,6 +6,7 @@ # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -64,6 +65,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/config-model/src/main/perl/expand-config.pl b/config-model/src/main/perl/expand-config.pl index 643de484620..198dc9f3447 100755 --- a/config-model/src/main/perl/expand-config.pl +++ b/config-model/src/main/perl/expand-config.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #============================================================================ diff --git a/config-model/src/main/perl/vespa-replicate-log-stream b/config-model/src/main/perl/vespa-replicate-log-stream index 3e02b498554..24081975e9b 100755 --- a/config-model/src/main/perl/vespa-replicate-log-stream +++ b/config-model/src/main/perl/vespa-replicate-log-stream @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl use IO::Socket::INET; diff --git a/config-model/src/main/resources/schema/search.rnc b/config-model/src/main/resources/schema/search.rnc index 19a3860d93b..a96a4576f7b 100644 --- a/config-model/src/main/resources/schema/search.rnc +++ b/config-model/src/main/resources/schema/search.rnc @@ -221,6 +221,9 @@ Tuning = element tuning { }? }? }? + }? & + element initialize { + element threads { xsd:nonNegativeInteger }? }? }? } diff --git a/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml b/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml index dbe3bb659e0..fe4942d7c97 100644 --- a/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml +++ b/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml @@ -1,10 +1,3 @@ <!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <jdisc> - <config name="cloud.config.elk"> - <elasticsearch> - <item> - <host>foo</host> - </item> - </elasticsearch> - </config> </jdisc> diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java index 35f823ecd30..be2bf690bdc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -9,7 +9,7 @@ import static org.junit.Assert.assertNotNull; /** * Tests that indexing statements are parsed correctly. * - * @author <a href="mailto:frodelu@yahoo-inc.com">Frode Lundgren</a> + * @author frodelu */ public class IndexingParsingTestCase extends SearchDefinitionTestCase { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 89f56470f1d..1c4564851d8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -13,7 +13,7 @@ import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTe /** * Test validation of changes between a current and next document type used in a document database. * - * @author <a href="mailto:Tor.Egge@yahoo-inc.com">Tor Egge</a> + * @author toregge * @since 2014-11-25 */ public class DocumentTypeChangeValidatorTest { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index caa4a34d47c..4a174970288 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -225,4 +225,14 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertThat(cfg, containsString("summary.log.chunk.compression.level 5")); } + @Test + public void requireThatWeCanParseInitializeTag() { + Tuning t = createTuning(parseXml("<initialize>", + "<threads>7</threads>", + "</initialize>")); + assertEquals(7, t.searchNode.initialize.threads.intValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("initialize.threads 7")); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java deleted file mode 100644 index 7a8a554e650..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.cloud.config.ElkConfig; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.configserver.TestOptions; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author lulf - * @since 5.17 - */ -public class ConfigServerContainerModelBuilderTest { - @Test - public void testHostedVespaInclude() { - File testApp = new File("src/test/cfg/container/data/configserverinclude"); - FilesApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - MockRoot root = new MockRoot(); - new ConfigServerContainerModelBuilder(new TestOptions()).build(new DeployState.Builder().applicationPackage(app).build(), null, root, XML.getChild(XML.getDocument(app.getServices()).getDocumentElement(), "jdisc")); - root.freezeModelTopology(); - ElkConfig config = root.getConfig(ElkConfig.class, "configserver/configserver"); - assertThat(config.elasticsearch().size(), is(1)); - assertThat(config.elasticsearch(0).host(), is("foo")); - } -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java index b0f2214d058..80fbcc71559 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java @@ -18,7 +18,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; /** - * @author hakon + * @author hakonhall */ public class StorageNodeTest { private StorDevicesConfig getConfig(boolean useVdsEngine) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index a86dc68d9dc..87fb5e567a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.config.UnknownConfigIdException; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -101,7 +102,7 @@ public class VespaModelTestCase { LogdConfig.Builder b = new LogdConfig.Builder(); b = (LogdConfig.Builder) model.getConfig(b, ""); LogdConfig c = new LogdConfig(b); - assertEquals(c.logserver().host(), LinuxInetAddress.getLocalHost().getCanonicalHostName()); + assertEquals(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()), c.logserver().host()); SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); sb = (com.yahoo.cloud.config.SlobroksConfig.Builder) model.getConfig(sb, ""); @@ -112,7 +113,7 @@ public class VespaModelTestCase { zb = (ZookeepersConfig.Builder) model.getConfig(zb, ""); ZookeepersConfig zc = new ZookeepersConfig(zb); assertEquals(zc.zookeeperserverlist().split(",").length, 2); - assertTrue(zc.zookeeperserverlist().startsWith(LinuxInetAddress.getLocalHost().getCanonicalHostName())); + assertTrue(zc.zookeeperserverlist().startsWith(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()))); ApplicationIdConfig.Builder appIdBuilder = new ApplicationIdConfig.Builder(); appIdBuilder = (ApplicationIdConfig.Builder) model.getConfig(appIdBuilder, ""); diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index d3d37adeb86..9e7f724da4c 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index c326e6d25c1..c068af49b5f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.inject.Inject; import com.yahoo.cloud.config.ApplicationIdConfig; /** diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index eba51af8882..21d3692634f 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-proxy</artifactId> <packaging>jar</packaging> diff --git a/config/.gitignore b/config/.gitignore index 170b98b585f..f2098c8a392 100644 --- a/config/.gitignore +++ b/config/.gitignore @@ -1,4 +1,3 @@ /pom.xml.build /target Makefile -Testing diff --git a/config/pom.xml b/config/pom.xml index 4dd4e1a37fd..ace03e11555 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config</artifactId> <packaging>container-plugin</packaging> diff --git a/config/src/apps/vespa-config/vespa-config.pl b/config/src/apps/vespa-config/vespa-config.pl index a87e5e52976..7bb9b9f20b3 100755 --- a/config/src/apps/vespa-config/vespa-config.pl +++ b/config/src/apps/vespa-config/vespa-config.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # # Various small functions used when bootstrapping the config system @@ -6,6 +6,7 @@ # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -64,6 +65,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index b79ff278e57..b8051fe1085 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -424,11 +424,7 @@ public class ConfigUtils { } public static String getCanonicalHostName() { - try { - return com.yahoo.net.LinuxInetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } + return com.yahoo.net.LinuxInetAddress.getLocalHost().getCanonicalHostName(); } /** diff --git a/configd/.gitignore b/configd/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/configd/.gitignore +++ b/configd/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/configd/src/apps/su/main.cpp b/configd/src/apps/su/main.cpp index 7baa726d630..4cce0b5ead0 100644 --- a/configd/src/apps/su/main.cpp +++ b/configd/src/apps/su/main.cpp @@ -7,18 +7,22 @@ /** * small utility to use instead of "su" when we want to just - * switch to the "yahoo" user without any more fuss + * switch to the vespa user without any more fuss **/ int main(int argc, char** argv) { if (argc < 2) { - fprintf(stderr, "missing arguments, usage: run-as-yahoo <cmd> [args ...]"); + fprintf(stderr, "missing arguments, usage: run-as-yahoo <cmd> [args ...]\n"); exit(1); } - struct passwd *p = getpwnam("yahoo"); + const char *username = getenv("VESPA_USER"); + if (username == NULL) { + username = "yahoo"; + } + struct passwd *p = getpwnam(username); if (p == NULL) { - perror("FATAL error: user 'yahoo' missing in passwd file"); + fprintf(stderr, "FATAL error: user '%s' missing in passwd file\n", username); exit(1); } gid_t g = p->pw_gid; diff --git a/configdefinitions/.gitignore b/configdefinitions/.gitignore index aa75679bac6..7f99529b963 100644 --- a/configdefinitions/.gitignore +++ b/configdefinitions/.gitignore @@ -2,4 +2,3 @@ configdefinitions.iml target /pom.xml.build Makefile -Testing diff --git a/configdefinitions/pom.xml b/configdefinitions/pom.xml index 8b50f7c8d3e..77ea6b55405 100644 --- a/configdefinitions/pom.xml +++ b/configdefinitions/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configdefinitions</artifactId> <packaging>container-plugin</packaging> diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 910c8da80ef..83e178bfa01 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -16,8 +16,6 @@ vespa_generate_config(configdefinitions configserver.def) install(FILES configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions dispatch.def) install(FILES dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) -vespa_generate_config(configdefinitions elk.def) -install(FILES elk.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions fleetcontroller.def) install(FILES fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions ilscripts.def) diff --git a/configdefinitions/src/vespa/elk.def b/configdefinitions/src/vespa/elk.def deleted file mode 100644 index 0b144e84293..00000000000 --- a/configdefinitions/src/vespa/elk.def +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -# elasticsearch: for future use -elasticsearch[].host string - -# Port 0 (default) means use the Defaults web service port -elasticsearch[].port int default=0 - -# logstash forwarder -logstash.network.servers[].host string -logstash.network.servers[].port int default=5043 -logstash.network.timeout int default=15 -logstash.files[].paths[] string -logstash.files[].fields{} string - -# A relative path will be prepended by vespa home. -# An absolute path will be used as-is. -logstash.config_file string default="conf/logstash-forwarder/config.json" - -logstash.source_field string default="index_source" -logstash.spool_size int default=100 -# Note: verbose=true actually means "not noisy" in the forwarder -logstash.forwarder_command_line_params string default = "-verbose=true" diff --git a/configgen/bin/make-config.pl b/configgen/bin/make-config.pl index 4629e6e3240..1837137ff07 100755 --- a/configgen/bin/make-config.pl +++ b/configgen/bin/make-config.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # # This script transforms a .def file into a .h and .cpp file for config @@ -71,6 +71,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/configgen/bin/make-configold.pl b/configgen/bin/make-configold.pl index 5ba2d88f67e..a4a4ad843d1 100755 --- a/configgen/bin/make-configold.pl +++ b/configgen/bin/make-configold.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # # This script transforms a .def file into a .h and .cpp file for config @@ -71,6 +71,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/configgen/pom.xml b/configgen/pom.xml index d8fc4703694..7981b22baa1 100644 --- a/configgen/pom.xml +++ b/configgen/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>configgen</artifactId> diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index 90d92c7c03e..fcc011790b9 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -1,7 +1,9 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_install_script(src/main/sh/cloudconfig_server-remove-state bin) -vespa_install_script(src/main/sh/start-filedistribution start-file libexec/vespa) +vespa_install_script(src/main/sh/start-filedistribution libexec/vespa) vespa_install_script(src/main/sh/ping-configserver libexec/vespa) vespa_install_script(src/main/sh/start-configserver libexec/vespa) vespa_install_script(src/main/sh/start-logd libexec/vespa) vespa_install_script(src/main/sh/stop-configserver libexec/vespa) +install(DIRECTORY src/main/resources/logd DESTINATION conf) +install(DIRECTORY src/main/resources/configserver-app DESTINATION conf) diff --git a/configserver/pom.xml b/configserver/pom.xml index 1bd08e49ec8..c98cc7e7b9c 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configserver</artifactId> <packaging>container-plugin</packaging> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java index 58020dd773c..1861965947c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.UnknownConfigIdException; import com.yahoo.vespa.config.protocol.*; import com.yahoo.vespa.config.util.ConfigUtils; -import java.net.UnknownHostException; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -154,13 +153,7 @@ class GetConfigProcessor implements Runnable { * Do not call java.net.Inet4AddressImpl.getLocalHostName() on each request, as this causes CPU bottlenecks. */ static { - String hostName = "unknown"; - try { - hostName = LinuxInetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - // ignore if it fails - } - localHostName = hostName; + localHostName = LinuxInetAddress.getLocalHost().getHostName(); } static boolean logDebug(Trace trace) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java index 53c54045bf6..a1c17a93022 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java @@ -20,7 +20,6 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.protocol.JRTConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; @@ -289,7 +288,6 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } RequestHandler handler = context.requestHandler(); return handler.resolveConfig(context.applicationId(), request, vespaVersion); - } protected Supervisor getSupervisor() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index f6b00440e30..4b94d4c21cc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import com.yahoo.cloud.config.ElkConfig; - /** * Controls the lifetime of the {@link SuperModel} and the {@link SuperModelRequestHandler}. * @@ -32,6 +30,7 @@ import com.yahoo.cloud.config.ElkConfig; * @since 5.9 */ public class SuperModelController implements RequestHandler { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelController.class.getName()); private volatile SuperModelRequestHandler handler; private final GenerationCounter generationCounter; @@ -39,14 +38,12 @@ public class SuperModelController implements RequestHandler { private final long masterGeneration; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; - private final ElkConfig elkConfig; private volatile boolean enabled = false; - public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig, ElkConfig elkConfig) { + public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig) { this.generationCounter = generationCounter; this.configDefinitionRepo = configDefinitionRepo; - this.elkConfig = elkConfig; this.masterGeneration = configserverConfig.masterGeneration(); this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); this.zone = new Zone(configserverConfig); @@ -85,7 +82,7 @@ public class SuperModelController implements RequestHandler { private SuperModelRequestHandler createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { long generation = generationCounter.get() + masterGeneration; - SuperModel model = new SuperModel(newModels, elkConfig, zone); + SuperModel model = new SuperModel(newModels, zone); return new SuperModelRequestHandler(model, configDefinitionRepo, generation, responseFactory); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java index a36167517d8..78b89d7a3da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java @@ -38,6 +38,7 @@ public class ApplicationConvergenceChecker extends AbstractComponent { private final static Set<String> serviceTypes = new HashSet<>(Arrays.asList( "container", + "container-clustercontroller", "qrserver", "docprocservice", "searchnode", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java index bd60527ca79..2ce95e016f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java @@ -23,7 +23,7 @@ import java.util.Optional; * Fetches log entries from logserver with level errors and fatal. The logserver only return * a log entry once over this API so doing repeated call will not give the same results. * - * @author dybdahl + * @author dybis */ public class LogServerLogGrabber extends AbstractComponent { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(LogServerLogGrabber.class.getName()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java deleted file mode 100644 index 318a3f81d52..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.model; -import com.yahoo.cloud.config.ElkConfig.Builder; - -import com.yahoo.cloud.config.ElkConfig; -import com.yahoo.vespa.defaults.Defaults; - -/** - * Produces the ELK config for the SuperModel - * - * @author vegardh - * @since 5.38 - * - */ -public class ElkProducer implements ElkConfig.Producer { - - private final ElkConfig config; - - public ElkProducer(ElkConfig config) { - this.config = config; - } - - @Override - public void getConfig(Builder builder) { - for (ElkConfig.Elasticsearch es : config.elasticsearch()) { - int port = es.port() != 0 ? es.port() : Defaults.getDefaults().vespaWebServicePort(); - builder.elasticsearch(new ElkConfig.Elasticsearch.Builder().host(es.host()).port(port)); - } - ElkConfig.Logstash.Builder logstashBuilder = new ElkConfig.Logstash.Builder(); - logstashBuilder. - config_file(Defaults.getDefaults().underVespaHome(config.logstash().config_file())). - source_field(config.logstash().source_field()). - spool_size(config.logstash().spool_size()); - ElkConfig.Logstash.Network.Builder networkBuilder = new ElkConfig.Logstash.Network.Builder(). - timeout(config.logstash().network().timeout()); - for (ElkConfig.Logstash.Network.Servers srv : config.logstash().network().servers()) { - networkBuilder. - servers(new ElkConfig.Logstash.Network.Servers.Builder(). - host(srv.host()). - port(srv.port())); - } - logstashBuilder.network(networkBuilder); - for (ElkConfig.Logstash.Files files : config.logstash().files()) { - logstashBuilder.files(new ElkConfig.Logstash.Files.Builder(). - paths(files.paths()). - fields(files.fields())); - } - builder.logstash(logstashBuilder); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java index 2be7860b01f..c6aa5ed7f8e 100755 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.model; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.RoutingConfig; import com.yahoo.config.ConfigInstance; @@ -22,19 +21,16 @@ import java.util.Map; * * @author vegardh * @since 5.9 - * */ -public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer, RoutingConfig.Producer { +public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Producer { private final Map<TenantName, Map<ApplicationId, Application>> models; private final LbServicesProducer lbProd; - private final ElkProducer elkProd; private final RoutingProducer zoneProd; - public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, ElkConfig elkConfig, Zone zone) { + public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, Zone zone) { this.models = newModels; this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); - this.elkProd = new ElkProducer(elkConfig); this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(models)); } @@ -44,16 +40,12 @@ public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); getConfig(builder); return ConfigPayload.fromInstance(new LbServicesConfig(builder)); - } else if (configKey.equals(new ConfigKey<>(ElkConfig.class, configKey.getConfigId()))) { - ElkConfig.Builder builder = new ElkConfig.Builder(); - getConfig(builder); - return ConfigPayload.fromInstance(new ElkConfig(builder)); } else if (configKey.equals(new ConfigKey<>(RoutingConfig.class, configKey.getConfigId()))) { RoutingConfig.Builder builder = new RoutingConfig.Builder(); getConfig(builder); return ConfigPayload.fromInstance(new RoutingConfig(builder)); } else { - return null; + throw new RuntimeException(configKey + " is not valid when asking for config from SuperModel"); } } @@ -67,11 +59,6 @@ public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer } @Override - public void getConfig(com.yahoo.cloud.config.ElkConfig.Builder builder) { - elkProd.getConfig(builder); - } - - @Override public void getConfig(RoutingConfig.Builder builder) { zoneProd.getConfig(builder); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index a69cf547db0..68bdf3150ab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; import java.util.Optional; -import com.yahoo.cloud.config.ElkConfig; - import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; @@ -43,8 +41,8 @@ public class SuperModelControllerTest { public void setup() throws IOException { counter = new SuperModelGenerationCounter(new MockCurator()); controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), - new ElkConfig(new ElkConfig.Builder())); + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder())); } @Test @@ -94,11 +92,11 @@ public class SuperModelControllerTest { TenantName tenantA = TenantName.from("a"); long masterGen = 10; controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)), - new ElkConfig(new ElkConfig.Builder())); + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen))); long gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3L, 1)); assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 93a48094fac..e5c839abb01 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -2,20 +2,16 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.LbServicesConfig; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.*; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; import com.yahoo.vespa.config.protocol.CompressionType; -import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -32,9 +28,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -import com.yahoo.cloud.config.ElkConfig.Logstash; - -import com.yahoo.vespa.config.server.model.ElkProducer; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -55,33 +48,7 @@ public class SuperModelRequestHandlerTest { ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); - handler = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); - } - - @Test - public void test_super_model_resolve_elk() { - ConfigResponse response = handler.resolveConfig(new GetConfigRequest() { - @Override - public ConfigKey<?> getConfigKey() { - return new ConfigKey<>(ElkConfig.class, "dontcare"); - } - - @Override - public DefContent getDefContent() { - return DefContent.fromClass(ElkConfig.class); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return Optional.empty(); - } - - @Override - public boolean noCache() { - return false; - } - }); - assertThat(response.getGeneration(), is(2l)); + handler = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @Test @@ -124,7 +91,7 @@ public class SuperModelRequestHandlerTest { models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); - SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); LbServicesConfig lbc = new LbServicesConfig(lb); @@ -157,47 +124,7 @@ public class SuperModelRequestHandlerTest { org.junit.Assert.fail("No qrserver service in config"); } - @Test - public void testElkConfig() { - ElkConfig ec = new ElkConfig(new ElkConfig.Builder().elasticsearch(new ElkConfig.Elasticsearch.Builder().host("es1").port(99)). - logstash(new ElkConfig.Logstash.Builder(). - config_file("/cfgfile"). - source_field("srcfield"). - spool_size(345). - network(new Logstash.Network.Builder(). - servers(new Logstash.Network.Servers.Builder(). - host("ls1"). - port(999)). - servers(new Logstash.Network.Servers.Builder(). - host("ls2"). - port(998)). - timeout(78)). - files(new ElkConfig.Logstash.Files.Builder(). - paths("path1"). - paths("path2"). - fields("field1", "f1val"). - fields("field2", "f2val")))); - ElkProducer ep = new ElkProducer(ec); - ElkConfig.Builder newBuilder = new ElkConfig.Builder(); - ep.getConfig(newBuilder); - ElkConfig elkConfig = new ElkConfig(newBuilder); - assertThat(elkConfig.elasticsearch(0).host(), is("es1")); - assertThat(elkConfig.elasticsearch(0).port(), is(99)); - assertThat(elkConfig.logstash().network().servers(0).host(), is("ls1")); - assertThat(elkConfig.logstash().network().servers(0).port(), is(999)); - assertThat(elkConfig.logstash().network().servers(1).host(), is("ls2")); - assertThat(elkConfig.logstash().network().servers(1).port(), is(998)); - assertThat(elkConfig.logstash().network().timeout(), is(78)); - assertThat(elkConfig.logstash().config_file(), is("/cfgfile")); - assertThat(elkConfig.logstash().source_field(), is("srcfield")); - assertThat(elkConfig.logstash().spool_size(), is(345)); - assertThat(elkConfig.logstash().files().size(), is(1)); - assertThat(elkConfig.logstash().files(0).paths(0), is("path1")); - assertThat(elkConfig.logstash().files(0).paths(1), is("path2")); - assertThat(elkConfig.logstash().files(0).fields("field1"), is("f1val")); - assertThat(elkConfig.logstash().files(0).fields("field2"), is("f2val")); - } - } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java index 3e5431deccd..30143f565e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; @@ -65,7 +64,7 @@ public class TestWithRpc { protected void createAndStartRpcServer(boolean hostedVespa) { rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), new SuperModelController(generationCounter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), new ElkConfig(new ElkConfig.Builder())), + new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder())), Metrics.createTestMetrics(), new HostRegistries()); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); diff --git a/configutil/.gitignore b/configutil/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/configutil/.gitignore +++ b/configutil/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index 8ff85cae6ce..ef8ea94abec 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-accesslogging</artifactId> <version>6-SNAPSHOT</version> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java index 987035bb0e9..f722511dd0c 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogSampler.java @@ -8,7 +8,7 @@ import java.util.concurrent.atomic.AtomicLong; * much less frequently to reduce CPU usage and latency impact. It only samples successful requests and requests * that starts with /search. * - * @author dybdahl + * @author dybis */ public class AccessLogSampler implements AccessLogInterface { diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java index 7a60eb2098c..fc458375d6a 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java @@ -9,7 +9,7 @@ import java.util.List; /** * This class keeps some information from the access log from the requests in memory. It is thread-safe. * - * @author dybdahl + * @author dybis */ public class CircularArrayAccessLogKeeper { public static final int SIZE = 1000; diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 5e86c55c294..2632c3d9aac 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -45,6 +45,7 @@ public class LogFileHandler extends StreamHandler { LogFileHandler logFileHandler; public LogThread(LogFileHandler logFile) { super("Logger"); + setDaemon(true); logFileHandler = logFile; } @Override diff --git a/container-core/pom.xml b/container-core/pom.xml index 650c5e87074..9932bc63c2b 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-core</artifactId> <version>6-SNAPSHOT</version> @@ -144,7 +143,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>log4j</groupId> diff --git a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java index 368a1f2cbf6..cc6ba83711d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java @@ -16,7 +16,7 @@ import java.util.concurrent.Executor; /** * Exposes access log through http. * - * @author dybdahl + * @author dybis */ public class AccessLogRequestHandler extends ThreadedHttpRequestHandler { private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper; diff --git a/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java b/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java index 64a75df5770..6b06b872770 100644 --- a/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java +++ b/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java @@ -43,13 +43,8 @@ public class ContainerRpcAdaptor extends AbstractRpcAdaptor { public ContainerRpcAdaptor(Osgi osgi) { this.osgi = osgi; - supervisor = new Supervisor(new Transport()); - - try { - this.hostname = LinuxInetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - throw new RuntimeException("Failed gettting local hostname", e); - } + this.supervisor = new Supervisor(new Transport()); + this.hostname = LinuxInetAddress.getLocalHost().getCanonicalHostName(); bindCommands(supervisor); } diff --git a/container-dev/pom.xml b/container-dev/pom.xml index ca21d564ed4..9b61e60d759 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-dev</artifactId> <version>6-SNAPSHOT</version> diff --git a/container-di/pom.xml b/container-di/pom.xml index 0cab901ab9f..181488a4e52 100644 --- a/container-di/pom.xml +++ b/container-di/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-di</artifactId> <version>6-SNAPSHOT</version> @@ -73,7 +72,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> <exclusions> <exclusion> <groupId>log4j</groupId> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index b94b626da73..c484cf98dc5 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-disc</artifactId> <version>6-SNAPSHOT</version> @@ -94,7 +93,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh new file mode 100755 index 00000000000..f76f8d70455 --- /dev/null +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -0,0 +1,159 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#set -x + +if [ -z "${VESPA_HOME}" ]; then + echo "Missing VESPA_HOME variable" + exit 1 +fi +if [ -z "${VESPA_SERVICE_NAME}" ]; then + echo "Missing VESPA_SERVICE_NAME variable" + exit 1 +fi +if [ -z "${VESPA_CONFIG_ID}" ]; then + echo "Missing VESPA_CONFIG_ID variable" + exit 1 +fi +cd ${VESPA_HOME} || { echo "Cannot cd to ${VESPA_HOME}" 1>&2; exit 1; } + +. libexec/vespa/common-env.sh + +DISCRIMINATOR=`echo ${VESPA_CONFIG_ID} | md5sum | cut -d' ' -f1` +CONTAINER_HOME="${VESPA_HOME}var/jdisc_container/${DISCRIMINATOR}/" + +ZOOKEEPER_LOG_FILE="${VESPA_HOME}logs/vespa/zookeeper.${VESPA_SERVICE_NAME}.log" +rm -f $ZOOKEEPER_LOG_FILE*lck + +# common setup +export VESPA_LOG_TARGET=file:${VESPA_HOME}logs/vespa/vespa.log +export VESPA_LOG_CONTROL_DIR=${VESPA_HOME}var/db/vespa/logcontrol +export LD_LIBRARY_PATH=${VESPA_HOME}lib64 + +cfpfile=${CONTAINER_HOME}/jdisc.properties +bundlecachedir=${CONTAINER_HOME}/bundlecache + +# class path +CP="${VESPA_HOME}lib/jars/jdisc_core-jar-with-dependencies.jar" + +mkdir -p $bundlecachedir || exit 1 +printenv > $cfpfile || exit 1 + +# ??? TODO ??? XXX ??? +# LANG=en_US.utf8 +# LC_ALL=C + + +getconfig() { + qrstartcfg="" + case "${VESPA_CONFIG_ID}" in + dir:*) + config_dir=${VESPA_CONFIG_ID#dir:} + qrstartcfg="`cat ${config_dir}/qr-start.cfg`" + ;; + *) + qrstartcfg="`getvespaconfig -w 10 -n search.config.qr-start -i ${VESPA_CONFIG_ID}`" + ;; + esac + cmds=`echo "$qrstartcfg" | perl -ne 's/^(\w+)\.(\w+) (.*)/$1_$2=$3/ && print'` + eval "$cmds" +} + +configure_memory() { + consider_fallback jvm_heapsize 1536 + consider_fallback jvm_stacksize 512 + consider_fallback jvm_baseMaxDirectMemorySize 75 + consider_fallback jvm_directMemorySizeCache 0 + + if (( jvm_heapSizeAsPercentageOfPhysicalMemory > 0 && jvm_heapSizeAsPercentageOfPhysicalMemory < 100 )); then + available=`free -m | grep Mem | tr -s ' ' | cut -f2 -d' '` + jvm_heapsize=$[available * jvm_heapSizeAsPercentageOfPhysicalMemory / 100] + if (( jvm_heapsize < 1024 )); then + jvm_heapsize=1024 + fi + fi + maxDirectMemorySize=$(( ${jvm_baseMaxDirectMemorySize} + ${jvm_heapsize}/8 + ${jvm_directMemorySizeCache} )) + + memory_options="-Xms${jvm_heapsize}m -Xmx${jvm_heapsize}m" + memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}" + memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + + if [ "${VESPA_USE_HUGEPAGES}" ]; then + memory_options="${memory_options} -XX:+UseLargePages" + fi +} + +configure_gcopts() { + consider_fallback jvm_gcopts "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" + if [ "$jvm_verbosegc" = "true" ]; then + jvm_gcopts="${jvm_gcopts} -verbose:gc" + fi +} + +configure_env_vars() { + if [ "$qrs_env" ]; then + for setting in ${qrs_env} ; do + case $setting in + *"="*) + eval "$setting"; + export ${setting%%=*} + ;; + *) + echo "warning ignoring invalid qrs_env setting '$setting' from '$qrs_env'" + ;; + esac + done + fi +} + +configure_classpath () { + if [ "${jdisc_classpath_extra}" ]; then + CP="${CP}:${jdisc_classpath_extra}" + fi +} + +configure_preload () { + export JAVAVM_LD_PRELOAD= + unset LD_PRELOAD + if [ "$PRELOAD" ]; then + export JAVAVM_LD_PRELOAD="$PRELOAD" + export LD_PRELOAD="$PRELOAD" + fi +} + +getconfig +configure_memory +configure_gcopts +configure_env_vars +configure_classpath +# note: should be last thing here: +configure_preload + +exec java \ + -Dconfig.id="${VESPA_CONFIG_ID}" \ + ${memory_options} \ + ${jvm_gcopts} \ + -XX:MaxJavaStackTraceDepth=-1 \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="${VESPA_HOME}var/crash" \ + -XX:OnOutOfMemoryError='kill -9 %p' \ + -Djava.library.path="${VESPA_HOME}lib64" \ + -Djava.awt.headless=true \ + -Djavax.net.ssl.keyStoreType=JKS \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 \ + -Djdisc.config.file="$cfpfile" \ + -Djdisc.export.packages=${jdisc_export_packages} \ + -Djdisc.cache.path="$bundlecachedir" \ + -Djdisc.debug.resources=false \ + -Djdisc.bundle.path="${VESPA_HOME}lib/jars" \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=ALL \ + -Djdisc.logger.tag="${VESPA_CONFIG_ID}" \ + -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger \ + -Dvespa.log.control.dir="${VESPA_LOG_CONTROL_DIR}" \ + -Dzookeeperlogfile="${ZOOKEEPER_LOG_FILE}" \ + -Dfile.encoding=UTF-8 \ + -cp "$CP" \ + "$@" \ + com.yahoo.jdisc.core.StandaloneMain file:${VESPA_HOME}lib/jars/container-disc-jar-with-dependencies.jar diff --git a/container-jersey2/pom.xml b/container-jersey2/pom.xml index f8e8b7f7d67..50adb243e93 100644 --- a/container-jersey2/pom.xml +++ b/container-jersey2/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-jersey2</artifactId> <version>6-SNAPSHOT</version> diff --git a/container-messagebus/pom.xml b/container-messagebus/pom.xml index 553f3bc0d8a..6e802e4a804 100644 --- a/container-messagebus/pom.xml +++ b/container-messagebus/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-messagebus</artifactId> <version>6-SNAPSHOT</version> diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 8c567c189ac..b4da1254ae0 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search-and-docproc</artifactId> <version>6-SNAPSHOT</version> diff --git a/container-search/pom.xml b/container-search/pom.xml index eaab9079b89..b632d022922 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search</artifactId> <packaging>jar</packaging> diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java deleted file mode 100644 index a282dc22b53..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.document.BucketId; -import com.yahoo.document.BucketIdFactory; -import com.yahoo.document.DocumentId; -import com.yahoo.document.GlobalId; -import com.yahoo.document.idstring.IdString; -import com.yahoo.documentapi.messagebus.protocol.SearchColumnPolicy; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.grouping.vespa.GroupingExecutor; -import com.yahoo.search.query.Model; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.result.DefaultErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.BucketDistribution; -import com.yahoo.component.chain.dependencies.Before; - -import java.util.Iterator; -import java.util.logging.Logger; - - -/** - * Searcher that does efficient key/value lookup using Vespa search as a - * backend. It does so by bypassing the first phase ranking, and only performs - * the second phase summary fetching. - * - * The keys to find are input as a comma-seprated list using the <i>keys</i> - * query parameter. Each key should match a part of a document id. Given the key - * 'foo', and document id namespace 'mynamespace', the document id matched will - * be 'id:mynamespace:keyvalue::foo'. - * - * To scale the throughput with the number of partitions, the searcher uses the - * same hashing mechanisms as the document API to find out which node each key - * belongs to. The searcher then dispatches a summary request to retrieve keys - * and returns the result. - * - * @author <a href="lulf@yahoo-inc.com">Ulf Lilleengen</a> - */ -@Before(GroupingExecutor.COMPONENT_NAME) -public class KeyValueSearcher extends Searcher { - - private static final Logger log = Logger.getLogger(KeyValueSearcher.class.getName()); - private final BucketIdFactory factory = new BucketIdFactory(); - private final BucketDistribution distribution; - private final String summaryClass; - private final String idSchemePrefix; - private final int numRowBits; - private final int traceLevel = 5; - - public KeyValueSearcher(KeyvalueConfig config) { - this.summaryClass = config.summaryName(); - this.idSchemePrefix = createIdSchemePrefix(config); - this.distribution = new BucketDistribution(config.numparts(), SearchColumnPolicy.DEFAULT_NUM_BUCKET_BITS); - this.numRowBits = calcNumRowBits(config.numrows()); - log.config("Configuring " + KeyValueSearcher.class.getName() + " with " + config.numparts() + " partitions and doc id scheme '" + idSchemePrefix + "'"); - } - - private String createIdSchemePrefix(KeyvalueConfig config) { - if (config.docIdScheme().equals(KeyvalueConfig.DocIdScheme.Enum.DOC_SCHEME)) { - return "doc:" + config.docIdNameSpace() + ":"; - } else { - return "id:" + config.docIdNameSpace() + ":" + config.docIdType() + "::"; - } - } - - public Hit createHit(Query query, String key) { - String docId = createDocId(key.trim()); - BucketId id = factory.getBucketId(new DocumentId(docId)); - int partition = getPartition(id); - - FastHit hit = new FastHit(); - hit.setGlobalId(new GlobalId(IdString.createIdString(docId))); - hit.setQuery(query); - hit.setFillable(); - hit.setCached(false); - hit.setPartId(partition << numRowBits, numRowBits); - hit.setRelevance(1.0); - hit.setIgnoreRowBits(true); - hit.setDistributionKey(42); - return hit; - } - - private String createDocId(String key) { - return idSchemePrefix + key; - } - - - @Override - public Result search(Query query, Execution execution) { - String keyProp = query.properties().getString("keys"); - query.getPresentation().setSummary(summaryClass); - if (keyProp == null || keyProp.length() == 0) { - return new Result(query, new ErrorMessage(ErrorMessage.NULL_QUERY, "'keys' parameter not set or empty.")); - } - String[] keyList = keyProp.split(","); - Model model = query.getModel(); - QueryTree tree = model.getQueryTree(); - QueryCanonicalizer.canonicalize(tree); - if (tree.isEmpty()) { - tree.setRoot(new IntItem(String.valueOf(keyProp.hashCode()))); - } - - Result result = new Result(query); - for (String key : keyList) { - result.hits().add(createHit(query, key)); - } - execution.fill(result, summaryClass); - if (query.isTraceable(traceLevel)) { - traceResult(query, result); - } - int totalHits = 0; - Iterator<Hit> hitIterator = result.hits().iterator(); - while (hitIterator.hasNext()) { - Hit hit = hitIterator.next(); - if (hit.isFillable() && hit.isFilled(summaryClass)) { - totalHits++; - } else { - hitIterator.remove(); - } - } - if (totalHits != keyList.length) { - ErrorMessage error = new ErrorMessage(1, "Some keys could not be fetched"); - result.hits().setError(error); - } - result.setTotalHitCount(totalHits); - return result; - } - - private void traceResult(Query query, Result result) { - Iterator<Hit> hitIterator = result.hits().iterator(); - while (hitIterator.hasNext()) { - Hit hit = hitIterator.next(); - if (hit.isFillable() && hit.isFilled(summaryClass)) { - query.trace("Found filled hit: " + hit, traceLevel); - } else { - query.trace("Found hit that was not filled/fillable: " + hit, traceLevel); - } - } - query.trace("Error hit: " + result.hits().getErrorHit(), traceLevel); - } - - private int getPartition(BucketId bucketId) { - return distribution.getColumn(bucketId); - } - - private static int calcNumRowBits(int numRows) { - if (numRows < 1) { - throw new IllegalArgumentException(); - } - for (int i = 0; i < 30; ++i) { - if (numRows - 1 < 1 << i) { - return i; - } - } - return 31; - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 03e212fc854..7e2ab396817 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -28,7 +28,7 @@ import com.yahoo.prelude.Location; * internal units (no suffix), meter (m), kilometer (km) or miles (mi) * </ul> * - * @author Arne J + * @author arnej27959 */ @After(PhaseNames.RAW_QUERY) @Before(PhaseNames.TRANSFORMED_QUERY) diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index c487182c65d..2775ba33146 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -109,17 +109,18 @@ public class NGramSearcher extends Searcher { * @return the root of the query subtree produced by this, containing the split items */ protected Item splitToGrams(Item term, String text, int gramSize, Query query) { - CompositeItem and = createGramRoot(query); String index = ((HasIndexItem)term).getIndexName(); + CompositeItem gramsItem = createGramRoot(query); + gramsItem.setIndexName(index); Substring origin = ((BlockItem)term).getOrigin(); for (Iterator<GramSplitter.Gram> i = getGramSplitter().split(text,gramSize); i.hasNext(); ) { GramSplitter.Gram gram = i.next(); WordItem gramWord = new WordItem(gram.extractFrom(text), index, false, origin); gramWord.setWeight(term.getWeight()); gramWord.setProtected(true); - and.addItem(gramWord); + gramsItem.addItem(gramWord); } - return and.getItemCount()==1 ? and.getItem(0) : and; // return the AndItem, or just the single gram if not multiple + return gramsItem.getItemCount()==1 ? gramsItem.getItem(0) : gramsItem; // return the AndItem, or just the single gram if not multiple } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index f6e8ee1f27a..653589a8e91 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -25,8 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull; * Ensure the fields specified in {@link Presentation#getSummaryFields()} are * available after filling phase. * - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author stiankri + * @author Steinar Knutsen */ @Beta @After(MinimalQueryInserter.EXTERNAL_YQL) diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index a7cc06c95f7..bcdc6ba060f 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -84,9 +84,9 @@ import edu.umd.cs.findbugs.annotations.NonNull; * VespaSerializer. * </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Steinar Knutsen + * @author stiankri + * @author Simon Thoresen */ @Beta public class YqlParser implements Parser { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java deleted file mode 100644 index 6a329185ef1..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.document.GlobalId; -import com.yahoo.document.idstring.IdString; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.searcher.KeyValueSearcher; -import com.yahoo.prelude.searcher.KeyvalueConfig; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import org.junit.Before; -import org.junit.Test; - -import java.util.*; -import java.util.Map.Entry; - -import static org.junit.Assert.*; - -public class KeyValueSearcherTest { - - private static class BackendMockup extends Searcher { - private final Map<GlobalId, Entry<String, String>> dataMap; - private final String summaryType; - - public BackendMockup(Map<GlobalId, Entry<String, String>> dataMap, String summaryType) { - this.dataMap = dataMap; - this.summaryType = summaryType; - } - - @Override - public Result search(Query query, Execution execution) { - fail("Should not do search against backend"); - return null; - } - - @Override - public void fill(Result result, String summaryClass, Execution execution) { - if (containsNullItem(result.getQuery().getModel().getQueryTree().getRoot())) - fail("Got a query with a NullItem root. This cannot be encoded."); - int numEmpty = 0; - for (Hit hit : result.hits()) { - FastHit fhit = (FastHit) hit; - Entry<String, String> data = dataMap.get(fhit.getGlobalId()); - if (data != null) { - fhit.setField(data.getKey(), data.getValue()); - fhit.setFilled(summaryType); - } else { - numEmpty++; - } - } - if (numEmpty > 0) { - result.hits().addError(ErrorMessage.createBackendCommunicationError("One or more hits were not filled")); - } - } - } - - private Map<GlobalId, Entry<String,String>> dataMap; - private BackendMockup backend; - @Before - public void setupBackend() { - dataMap = new HashMap<>(); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::foo")), new AbstractMap.SimpleEntry<>("foo", "foovalue")); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::bar")), new AbstractMap.SimpleEntry<>("bar", "barvalue")); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::this_must_be_a_key_in_part1_fsadfasdfa")), new AbstractMap.SimpleEntry<>("this_must_be_a_key_in_part1_fsadfasdfa", "blabla")); - backend = new BackendMockup(dataMap, "mysummary"); - } - - @Test - public void testKeyValueSearcher() { - Result result = executeQuery(getConfigString(1), "?keys=foo,bar"); - assertEquals(2, result.getTotalHitCount()); - for (Hit hit : result.hits()) { - FastHit fhit = (FastHit)hit; - Entry<String, String> data = dataMap.get(fhit.getGlobalId()); - assertEquals(data.getValue(), hit.getField(data.getKey())); - assertTrue(hit.isFilled("mysummary")); - } - - result = executeQuery(getConfigString(1), - "?keys=blabla,fofo", new BackendMockup(dataMap, "mysummary")); - assertEquals(0, result.getTotalHitCount()); - - result = executeQuery(getConfigString(1), - "?keys=non,foo,slsl", new BackendMockup(dataMap, "mysummary")); - assertEquals(1, result.getTotalHitCount()); - } - - @Test - public void testKeyValueSearcherWithNullItemAsQuery() { - Query query = new Query("?keys=foo,bar"); - AndItem and = new AndItem(); - and.addItem(new NullItem()); - query.getModel().getQueryTree().setRoot(and); - Result result = executeQuery(getConfigString(1), query); - assertEquals(2, result.getTotalHitCount()); - } - - private static String getConfigString(int numRows) { - return "raw:numparts 2\nsummaryName \"mysummary\"\ndocIdType \"keyvalue\"\ndocIdNameSpace \"keyvalue\"\nnumrows " + numRows + "\n"; - } - - @Test - public void requireThatIgnoreRowBitsIsEnabledInGeneratedHits() { - Result result = executeQuery(getConfigString(1), - "?keys=foo,bar"); - for (Hit hit : result.hits()) { - FastHit fastHit = (FastHit)hit; - assertTrue(fastHit.shouldIgnoreRowBits()); - } - } - - @Test - public void requireThatNumRowsIsAPositiveNumber() { - for (int i = -10; i < 1; ++i) { - try { - newKeyValueSearcher(getConfigString(i)); - fail(); - } catch (IllegalArgumentException e) { - - } - } - for (int i = 1; i < 10; ++i) { - assertNotNull(newKeyValueSearcher(getConfigString(i))); - } - } - - @Test - public void requireThatNumRowBitsAreCalculatedCorrectly() { - assertRowBits(1, 0); - assertRowBits(2, 1); - assertRowBits(3, 2); - assertRowBits(4, 2); - assertRowBits(5, 3); - assertRowBits(10, 4); - assertRowBits(100, 7); - assertRowBits(1000, 10); - } - - private void assertRowBits(int numRows, int expectedNumRowBits) { - Result result = executeQuery(getConfigString(numRows), "?keys=this_must_be_a_key_in_part1_fsadfasdfa"); - assertEquals(1, result.hits().size()); - FastHit hit = (FastHit)result.hits().get(0); - assertEquals(0, hit.getPartId() & ((1 << expectedNumRowBits) - 1)); - assertEquals(1, hit.getPartId() >> expectedNumRowBits); - } - - private Result executeQuery(String configId, String queryString, Searcher... searchers) { - return executeQuery(configId, new Query(queryString), searchers); - } - - private Result executeQuery(String configId, Query query, Searcher... searchers) { - List<Searcher> chain = new LinkedList<>(); - chain.add(newKeyValueSearcher(configId)); - chain.addAll(Arrays.asList(searchers)); - chain.add(backend); - return new Execution(new Chain<>(chain), Execution.Context.createContextStub()).search(query); - } - - - private static KeyValueSearcher newKeyValueSearcher(String configId) { - return new KeyValueSearcher(new ConfigGetter<>(KeyvalueConfig.class).getConfig(configId)); - } - - private static boolean containsNullItem(Item item) { - if (item instanceof NullItem) return true; - if (item instanceof CompositeItem) { - for (Iterator<Item> i = ((CompositeItem)item).getItemIterator(); i.hasNext(); ) - if (containsNullItem(i.next())) - return true; - } - return false; - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java index c674a8a0755..135e038f800 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** - * @author <a href="http://techyard.corp.yahoo-inc.com/en/user/andreer">Andreas Eriksen</a> + * @author andreer */ public class UniqueGroupingSearcherTestCase { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java index b112e61cb44..23316a58877 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java @@ -17,7 +17,7 @@ import static com.yahoo.search.searchers.CacheControlSearcher.CACHE_CONTROL_HEAD /** * Unit test cases for CacheControlSearcher. * - * @author <a href="http://techyard.corp.yahoo-inc.com/en/user/frodelu">Frode Lundgren</a> + * @author frodelu */ @SuppressWarnings("deprecation") public class CacheControlSearcherTestCase extends TestCase { diff --git a/container-test-jars/pom.xml b/container-test-jars/pom.xml index 9d14c4b3de2..2cc9ee4d67f 100644 --- a/container-test-jars/pom.xml +++ b/container-test-jars/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.container.container-test-jars</groupId> <artifactId>container-test-jars</artifactId> diff --git a/defaults/.gitignore b/defaults/.gitignore index d2a86c8103d..12d711807b0 100644 --- a/defaults/.gitignore +++ b/defaults/.gitignore @@ -10,4 +10,3 @@ target *.iws /pom.xml.build Makefile -Testing diff --git a/defaults/pom.xml b/defaults/pom.xml index 7909947416a..17ca6bff3c3 100644 --- a/defaults/pom.xml +++ b/defaults/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> diff --git a/dist/vespa.spec b/dist/vespa.spec index 9effb0c0571..751d23df859 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -8,34 +8,44 @@ Name: vespa Version: VESPA_VERSION Release: 1%{?dist} -Summary: Vespa - +Summary: Vespa - The open big data serving engine Group: Applications/Databases License: Commercial URL: http://vespa.corp.yahoo.com Source0: vespa-%{version}.tar.gz - -#BuildRequires: vespa-boost-devel >= 1.59 -#BuildRequires: vespa-cppunit-devel >= 1.12.1 -#BuildRequires: vespa-libtorrent-devel >= 1.0.9 -#BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.8 -#BuildRequires: cmake3 >= 3.5 -#BuildRequires: epel-release -#BuildRequires: centos-release-scl -#BuildRequires: devtoolset-4 >= 4.0 -#BuildRequires: devtoolset-4-libatomic-devel -#BuildRequires: Judy-devel >= 1.0.5 -#BuildRequires: lz4-devel >= r131 -#BuildRequires: maven >= 3.0 -#BuildRequires: libicu-devel >= 50.1.2 -#BuildRequires: llvm-devel >= 3.4.2 -#BuildRequires: llvm-static >= 3.4.2 -#Requires: vespa-boost -#Requires: vespa-cppunit -#Requires: vespa-libtorrent -#Requires: vespa-zookeeper-c-client -#Requires: numactl +BuildRequires: epel-release +BuildRequires: centos-release-scl +BuildRequires: devtoolset-4-gcc-c++ +BuildRequires: devtoolset-4-libatomic-devel +BuildRequires: Judy-devel +BuildRequires: cmake3 +BuildRequires: lz4-devel +BuildRequires: zlib-devel +BuildRequires: maven +BuildRequires: libicu-devel +BuildRequires: llvm-devel +BuildRequires: java-1.8.0-openjdk-devel +BuildRequires: openssl-devel +BuildRequires: rpm-build +BuildRequires: make +BuildRequires: vespa-boost-devel >= 1.59 +BuildRequires: vespa-cppunit-devel >= 1.12.1 +BuildRequires: vespa-libtorrent-devel >= 1.0.9 +BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.8 +BuildRequires: systemd +Requires: epel-release +Requires: Judy +Requires: lz4 +Requires: zlib +Requires: libicu +Requires: llvm +Requires: java-1.8.0-openjdk +Requires: openssl +Requires: vespa-boost >= 1.59 +Requires: vespa-cppunit >= 1.12.1 +Requires: vespa-libtorrent >= 1.0.9 +Requires: vespa-zookeeper-c-client >= 3.4.8 Requires(pre): shadow-utils # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private @@ -44,33 +54,31 @@ Provides: libc.so.6(GLIBC_PRIVATE)(64bit) %description -This is the Vespa! +Vespa - The open big data serving engine %prep %setup -q %build - source /opt/rh/devtoolset-4/enable || true sh bootstrap.sh +mvn install -DskipTests -Dmaven.javadoc.skip=true cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm" \ -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include" \ - -DCMAKE_INSTALL_RPATH=%{_prefix}/lib64 \ + -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ -DCMAKE_BUILD_RPATH=%{_prefix}/lib64 \ . make %{_smp_mflags} -mvn install -DskipTests -Dmaven.javadoc.skip=true %install - rm -rf $RPM_BUILD_ROOT make install DESTDIR=%{buildroot} +# BEGIN - Put this in post install script called by make install # Rewrite config def file names - for path in %{buildroot}/%{_prefix}var/db/vespa/config_server/serverdb/classes/*.def; do dir=$(dirname $path) filename=$(basename $path) @@ -114,82 +122,59 @@ mkdir -p %{buildroot}/%{_prefix}/var/db/vespa/config_server/serverdb/application mkdir -p %{buildroot}/%{_prefix}/var/db/vespa/logcontrol/ mkdir -p %{buildroot}/%{_prefix}/var/jdisc_container/ mkdir -p %{buildroot}/%{_prefix}/var/jdisc_core/ +mkdir -p %{buildroot}/%{_prefix}/var/run/ mkdir -p %{buildroot}/%{_prefix}/var/spool/vespa/ mkdir -p %{buildroot}/%{_prefix}/var/spool/master/inbox/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/bundlecache/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/cache/config/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/cmdlines/ -mkdir -p %{buildroot}/%{_prefix}/var/zookeeper/ +mkdir -p %{buildroot}/%{_prefix}/var/zookeeper/version-2/ -%clean +ln -s %{_prefix}/lib/jars/config-model-fat.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/config-model-fat.jar +ln -s %{_prefix}/lib/jars/configserver-jar-with-dependencies.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/configserver.jar +ln -s %{_prefix}/lib/jars/orchestrator-jar-with-dependencies.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/orchestrator.jar +ln -s %{_prefix}/lib/jars/node-repository-jar-with-dependencies.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/node-repository.jar +ln -s %{_prefix}/lib/jars/zkfacade-jar-with-dependencies.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/zkfacade.jar +ln -s %{_prefix}/conf/configserver-app/components %{buildroot}/%{_prefix}/lib/jars/config-models +ln -s storaged-bin %{buildroot}/%{_prefix}/sbin/distributord-bin +# END - Put this in post install script called by make install + +mkdir -p %{buildroot}/usr/lib/systemd/system +cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib/systemd/system +cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system +%clean rm -rf $RPM_BUILD_ROOT %pre getent group vespa >/dev/null || groupadd -r vespa getent passwd vespa >/dev/null || \ - useradd -r -g vespa -d /opt/vespa -s /sbin/nologin \ + useradd -r -g vespa -d %{_prefix} -s /sbin/nologin \ -c "Create owner of all Vespa data files" vespa +echo "pathmunge %{_prefix}/bin" > /etc/profile.d/vespa.sh +echo "export VESPA_HOME=%{_prefix}" >> /etc/profile.d/vespa.sh +chmod +x /etc/profile.d/vespa.sh exit 0 +%post +%systemd_post vespa-configserver.service +%systemd_post vespa.service + +%preun +%systemd_preun vespa.service +%systemd_preun vespa-configserver.service + +%postun +%systemd_postun_with_restart vespa.service +%systemd_postun_with_restart vespa-configserver.service +rm -f /etc/profile.d/vespa.sh +userdel vespa %files -%defattr(-,root,root,-) +%defattr(-,vespa,vespa,-) %doc - -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/configserver/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/configserver-app/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/configserver-app/config-models/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/configserver-app/components/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/filedistributor/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/node-admin-app/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/node-admin-app/components/ -%dir %attr( 755, vespa, vespa) %{_prefix}/conf/zookeeper/ -%dir %attr( 777, -, -) %{_prefix}/libexec/jdisc_core/ -%dir %attr( 775, vespa, vespa) %{_prefix}/libexec/vespa/modelplugins/ -%dir %attr( 755, vespa, vespa) %{_prefix}/libexec/vespa/plugins/qrs/ -%dir %attr( 755, vespa, vespa) %{_prefix}/libexec/yjava_daemon/bin/ -%dir %attr( 777, vespa, vespa) %{_prefix}/logs/jdisc_core/ -%dir %attr(1777, vespa, vespa) %{_prefix}/logs/vespa/ -%dir %attr(1777, vespa, vespa) %{_prefix}/logs/vespa/ -%dir %attr( 755, vespa, vespa) %{_prefix}/logs/vespa/configserver/ -%dir %attr( 755, vespa, vespa) %{_prefix}/logs/vespa/search/ -%dir %attr( 755, vespa, vespa) %{_prefix}/logs/vespa/qrs/ -%dir %attr( 755, vespa, vespa) %{_prefix}/share/vespa/ -%dir %attr( 755, vespa, vespa) %{_prefix}/share/vespa/schema/version/6.x/schema/ -%dir %attr(1777, vespa, vespa) %{_prefix}/tmp/vespa/ -%dir %attr( 777, vespa, vespa) %{_prefix}/var/db/jdisc/logcontrol/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/db/vespa/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/db/vespa/config_server/serverdb/configs/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/db/vespa/config_server/serverdb/configs/application/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/db/vespa/config_server/serverdb/applications/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/db/vespa/logcontrol/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/jdisc_container/ -%dir %attr( 777, -, -) %{_prefix}/var/jdisc_core/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/spool/vespa/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/spool/master/inbox/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/bundlecache/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/cache/config/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/cmdlines/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/zookeeper/ - -%{_prefix}/libexec/vespa/vespa-config.pl -%{_prefix}/libexec/vespa/common-env.sh -%{_prefix}/libexec/vespa/start-vespa-base.sh -%{_prefix}/libexec/vespa/stop-vespa-base.sh -%{_prefix}/libexec/vespa/start-file -%{_prefix}/libexec/vespa/ping-configserver -%{_prefix}/libexec/vespa/start-configserver -%{_prefix}/libexec/vespa/start-logd -%{_prefix}/libexec/vespa/stop-configserver -%{_prefix}/var/db/vespa/config_server/serverdb/classes/*.def -%{_prefix}/lib/jars/*.jar -%{_prefix}/lib/perl5/site_perl/Yahoo/Vespa/*.pm -%{_prefix}/lib64/*.so -%{_prefix}/bin/* -%{_prefix}/sbin/* -%{_prefix}/man/* -%{_prefix}/include/* -%{_prefix}/etc/* +%{_prefix}/* +%attr(644,root,root) /usr/lib/systemd/system/vespa.service +%attr(644,root,root) /usr/lib/systemd/system/vespa-configserver.service %changelog diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build new file mode 100644 index 00000000000..221d1a7825c --- /dev/null +++ b/docker/Dockerfile.build @@ -0,0 +1,38 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install devtoolset-4-gcc-c++ +RUN yum -y install devtoolset-4-libatomic-devel +RUN yum -y install make +RUN yum -y install cmake3 +RUN yum -y install ccache +RUN yum -y install Judy-devel +RUN yum -y install lz4-devel +RUN yum -y install zlib-devel +RUN yum -y install maven +RUN yum -y install libicu-devel +RUN yum -y install llvm-devel +RUN yum -y install llvm-static +RUN yum -y install java-1.8.0-openjdk-devel +RUN yum -y install openssl +RUN yum -y install openssl-devel +RUN yum -y install rpm-build +RUN yum -y install perl +RUN yum -y install perl-Env +RUN yum -y install perl-JSON + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost-devel +RUN yum -y install vespa-libtorrent-devel +RUN yum -y install vespa-zookeeper-c-client-devel +RUN yum -y install vespa-cppunit-devel + +# Install utilities +RUN yum -y install sudo + +# Enable devtoolset-4 by default +RUN echo "source /opt/rh/devtoolset-4/enable" > /etc/profile.d/devtoolset-4.sh + diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run new file mode 100644 index 00000000000..d82297ce676 --- /dev/null +++ b/docker/Dockerfile.run @@ -0,0 +1,27 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install Judy +RUN yum -y install lz4 +RUN yum -y install zlib +RUN yum -y install libicu +RUN yum -y install llvm +RUN yum -y install java-1.8.0-openjdk +RUN yum -y install openssl +RUN yum -y install perl +RUN yum -y install perl-Env +RUN yum -y install perl-JSON +RUN yum -y install libatomic + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost +RUN yum -y install vespa-libtorrent +RUN yum -y install vespa-zookeeper-c-client +RUN yum -y install vespa-cppunit # Should not be needed ? + +# Utilities +RUN yum -y install net-tools less + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..59424ff2d98 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,33 @@ + +# Building and running Vespa on Docker (OS X and Linux) + +## Installing docker +[Docker installation](https://docs.docker.com/engine/installation/) + +*On OS X, the native Docker engine (Beta) has NOT been tested. Please use the [Docker Toolbox](https://www.docker.com/products/docker-toolbox).* + +*On Linux, the default storage device is devicemapper with loopback device and max 10GB container size. This size is too small for a full build. Please see [here](http://www.projectatomic.io/blog/2016/03/daemon_option_basedevicesize/) and [here](http://www.projectatomic.io/blog/2015/06/notes-on-fedora-centos-and-docker-storage-drivers/) to overcome this limitation.* + +## Building the Vespa RPM +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this source code. + +The produced rpms will be available in this folder after compiliation. + +## Running Vespa +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa. + +This will create a Docker image which has the rpms from the build step (or downloaded rpms to this folder) installed. Vespa will be started inside the container. + +*On OS X, the container runs inside the Docker VM. Execute ```docker-machine ssh vespa-docker-machine``` to enter the VM. The services can also be reached directly from the host on the IP given by ```docker-machine ip vespa-docker-machine```* + +## Building Vespa inside a Docker container +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./enter-build-container.sh``` to enter the Vespa build environment inside a Docker container. + +The container is entered at the root of the Vespa source repository. Follow the build sections in [README.md](https://github.com/yahoo/vespa/blob/master/README.md) to build and test. + diff --git a/docker/build-vespa-internal.sh b/docker/build-vespa-internal.sh new file mode 100755 index 00000000000..f79e936c800 --- /dev/null +++ b/docker/build-vespa-internal.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [ $# -ne 3 ]; then + echo "Usage: $0 <vespa version> <caller uid> <caller gid>" + echo "This script should not be called manually." + exit 1 +fi +VESPA_VERSION=$1 +CALLER_UID=$2 +CALLER_GID=$3 + +cd /vespa +./dist.sh ${VESPA_VERSION} +rpmbuild -bb ~/rpmbuild/SPECS/vespa-${VESPA_VERSION}.spec +chown ${CALLER_UID}:${CALLER_GID} ~/rpmbuild/RPMS/x86_64/*.rpm +mv ~/rpmbuild/RPMS/x86_64/*.rpm /vespa/docker + diff --git a/docker/build-vespa.sh b/docker/build-vespa.sh new file mode 100755 index 00000000000..6d3b1699bc5 --- /dev/null +++ b/docker/build-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/build-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" "$(id -u)" "$(id -g)" + diff --git a/docker/enter-build-container-internal.sh b/docker/enter-build-container-internal.sh new file mode 100755 index 00000000000..7da96fde376 --- /dev/null +++ b/docker/enter-build-container-internal.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + echo "This script should not be called manually." + exit 1 +fi + +USERNAME=builder +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +CALLER_UID=$(stat -c "%u" $DIR) +CALLER_GID=$(stat -c "%g" $DIR) + +groupadd -f -g $CALLER_GID $USERNAME +useradd -u $CALLER_UID -g $CALLER_GID $USERNAME +echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +su -c "mkdir -p $DIR/../.ccache" $USERNAME +su -c "ln -sf $DIR/../.ccache /home/$USERNAME/.ccache" $USERNAME + +su -c "mkdir -p $DIR/../.m2" $USERNAME +su -c "ln -sf $DIR/../.m2 /home/$USERNAME/.m2" $USERNAME + +cd $DIR/.. +su $USERNAME + diff --git a/docker/enter-build-container.sh b/docker/enter-build-container.sh new file mode 100755 index 00000000000..ed7a2b4a130 --- /dev/null +++ b/docker/enter-build-container.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run -ti --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/enter-build-container-internal.sh "$DOCKER_IMAGE" + diff --git a/docker/osx-setup-docker-machine.sh b/docker/osx-setup-docker-machine.sh new file mode 100755 index 00000000000..f6c9e870cf8 --- /dev/null +++ b/docker/osx-setup-docker-machine.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_VM_NAME=vespa-docker-machine +DOCKER_VM_DISK_SIZE_IN_MB=40000 +DOCKER_VM_MEMORY_SIZE_IN_MB=4000 +DOCKER_VM_CPU_COUNT=4 + +DOCKER_VM_WAS_STARTED=false + +if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then + # Machine does not exist and we have to create and start + docker-machine create -d virtualbox \ + --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \ + --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \ + --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \ + "$DOCKER_VM_NAME" + + eval $(docker-machine env "$DOCKER_VM_NAME") + DOCKER_VM_WAS_STARTED=true +fi + + +VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +if [ "$VESPA_VM_STATUS" = "Stopped" ]; then + docker-machine start "$DOCKER_VM_NAME" + DOCKER_VM_WAS_STARTED=true + VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +fi + +if [ "$VESPA_VM_STATUS" != "Running" ]; then + echo "Unable to get Docker machine $DOCKER_VM_NAME up and running." + echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME " + echo " and then rerun this script." + echo "Exiting." + exit 1 +fi + +if $DOCKER_VM_WAS_STARTED; then + # Hostname should match the public IP + docker-machine ssh "$DOCKER_VM_NAME" "sudo sed -i \"s/127.0.0.1 $DOCKER_VM_NAME/127.0.0.1/\" /etc/hosts" + docker-machine ssh "$DOCKER_VM_NAME" "sudo sed -i \"/$DOCKER_VM_NAME/d\" /etc/hosts" + docker-machine ssh "$DOCKER_VM_NAME" "sudo echo $(docker-machine ip $DOCKER_VM_NAME) $DOCKER_VM_NAME | sudo tee -a /etc/hosts" > /dev/null +fi + +eval $(docker-machine env "$DOCKER_VM_NAME") + diff --git a/docker/run-vespa-internal.sh b/docker/run-vespa-internal.sh new file mode 100755 index 00000000000..f019f1a3740 --- /dev/null +++ b/docker/run-vespa-internal.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + echo "This script should not be called manually." + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 + +rpm -i "vespa*-${VESPA_VERSION}-*.rpm" + +# Workaround until we figure out why rpm does not set the ownership. +chown -R vespa:vespa /opt/vespa + +/opt/vespa/bin/vespa-start-configserver +/opt/vespa/bin/vespa-start-services + +# Print log forever +while true; do + /opt/vespa/bin/logfmt -f /opt/vespa/logs/vespa/vespa.log + sleep 10 +done diff --git a/docker/run-vespa.sh b/docker/run-vespa.sh new file mode 100755 index 00000000000..e11324a7b93 --- /dev/null +++ b/docker/run-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE=vesparun + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.run . +docker run -d -v $(pwd)/..:/vespa --net=host --privileged --entrypoint /vespa/docker/run-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" + diff --git a/docproc/pom.xml b/docproc/pom.xml index fe7228ba47d..8cec9b6a612 100644 --- a/docproc/pom.xml +++ b/docproc/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docproc</artifactId> <packaging>jar</packaging> diff --git a/docprocs/pom.xml b/docprocs/pom.xml index 64fed1d55e2..d85a4c475dd 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docprocs</artifactId> <packaging>container-plugin</packaging> diff --git a/document/.gitignore b/document/.gitignore index e23d1d9f4e9..7a18138fa03 100644 --- a/document/.gitignore +++ b/document/.gitignore @@ -11,4 +11,3 @@ libexec target /pom.xml.build Makefile -Testing diff --git a/document/pom.xml b/document/pom.xml index 5d882b17f25..9b096f3c89a 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>document</artifactId> <version>6-SNAPSHOT</version> diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java index 2b210cb2ee5..4476acfae22 100644 --- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java @@ -14,7 +14,7 @@ import java.io.InputStream; /** * Parser that supports parsing PUT operation and UPDATE operation. * - * @author dybdahl + * @author dybis */ public class SingleDocumentParser { private static final JsonFactory jsonFactory = new JsonFactory(); diff --git a/documentapi/.gitignore b/documentapi/.gitignore index 9690261769a..57e44062f77 100644 --- a/documentapi/.gitignore +++ b/documentapi/.gitignore @@ -2,4 +2,3 @@ documentapi.iml target /pom.xml.build Makefile -Testing diff --git a/documentapi/OWNERS b/documentapi/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/documentapi/OWNERS +++ b/documentapi/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/documentapi/pom.xml b/documentapi/pom.xml index 7f103965893..d9a3052def5 100644 --- a/documentapi/pom.xml +++ b/documentapi/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentapi</artifactId> <packaging>container-plugin</packaging> diff --git a/documentgen-test/OWNERS b/documentgen-test/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/documentgen-test/OWNERS +++ b/documentgen-test/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml index 3e32e62d17f..8387a586701 100644 --- a/documentgen-test/pom.xml +++ b/documentgen-test/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentgen-test</artifactId> <packaging>jar</packaging> diff --git a/dummy-persistence/pom.xml b/dummy-persistence/pom.xml index 3d3c398ab49..c831bbe61f8 100644 --- a/dummy-persistence/pom.xml +++ b/dummy-persistence/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>simple</groupId> <artifactId>dummy-persistence</artifactId> <!-- Also used as Bundle-SymbolicName --> diff --git a/fastlib/.gitignore b/fastlib/.gitignore index d94143dba02..d52d93b8dda 100644 --- a/fastlib/.gitignore +++ b/fastlib/.gitignore @@ -11,4 +11,3 @@ include lib update.log Makefile -Testing diff --git a/fastlib/OWNERS b/fastlib/OWNERS index 9e842aedad9..885ab949d74 100644 --- a/fastlib/OWNERS +++ b/fastlib/OWNERS @@ -1,3 +1,3 @@ havardpe -tegge +toregge baldersheim diff --git a/fastlib/src/vespa/fastlib/net/httpserver.cpp b/fastlib/src/vespa/fastlib/net/httpserver.cpp index 6d60b609b5f..e12477bd030 100644 --- a/fastlib/src/vespa/fastlib/net/httpserver.cpp +++ b/fastlib/src/vespa/fastlib/net/httpserver.cpp @@ -406,7 +406,7 @@ void Fast_HTTPServer::Stop(void) { _runningMutex.Lock(); _stopSignalled = true; - if (_isRunning) { + if (_acceptThread) { _acceptThread->SetBreakFlag(); } _runningMutex.Unlock(); diff --git a/fastos/.gitignore b/fastos/.gitignore index 70fbbb55a20..54e2680a6d8 100644 --- a/fastos/.gitignore +++ b/fastos/.gitignore @@ -9,4 +9,3 @@ include lib update.log Makefile -Testing diff --git a/fbench/.gitignore b/fbench/.gitignore index 069dac5c201..2b7330e0c48 100644 --- a/fbench/.gitignore +++ b/fbench/.gitignore @@ -4,4 +4,3 @@ doc include lib Makefile -Testing diff --git a/fileacquirer/pom.xml b/fileacquirer/pom.xml index 157ddf1f7d6..f6269b29b01 100644 --- a/fileacquirer/pom.xml +++ b/fileacquirer/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fileacquirer</artifactId> <version>6-SNAPSHOT</version> diff --git a/filedistribution/.gitignore b/filedistribution/.gitignore index 542ce6cebef..77f7a1d62f2 100644 --- a/filedistribution/.gitignore +++ b/filedistribution/.gitignore @@ -4,4 +4,3 @@ project.dsw /pom.xml.build /target Makefile -Testing diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index ea5fb5b450e..d6885bc0667 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>filedistribution</artifactId> <version>6-SNAPSHOT</version> diff --git a/filedistribution/src/vespa/filedistribution/common/vespa_logfwd.cpp b/filedistribution/src/vespa/filedistribution/common/vespa_logfwd.cpp index ab2a2334434..c45fbb84e56 100644 --- a/filedistribution/src/vespa/filedistribution/common/vespa_logfwd.cpp +++ b/filedistribution/src/vespa/filedistribution/common/vespa_logfwd.cpp @@ -42,6 +42,6 @@ void filedistribution::logfwd::log(LogLevel level, const char* file, int line, c vsnprintf(payload.get(), maxSize, fmt, args); va_end(args); - logger.doLog(vespaLogLevel, file, line, payload.get()); + logger.doLog(vespaLogLevel, file, line, "%s", payload.get()); } } diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp index 57fae9df6ee..1af58514fb9 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp @@ -141,7 +141,7 @@ struct FileDownloader::EventHandler BOOST_THROW_EXCEPTION(std::runtime_error(alert.message())); } void operator()(const libtorrent::fastresume_rejected_alert& alert) const { - LOG(info, "alert %s: %s", alert.what(), alert.message().c_str()); + LOG(debug, "alert %s: %s", alert.what(), alert.message().c_str()); } void operator()(const libtorrent::torrent_delete_failed_alert& alert) const { LOG(warning, "alert %s: %s", alert.what(), alert.message().c_str()); diff --git a/filedistribution_test/pom.xml b/filedistribution_test/pom.xml index db6584b24b5..ab43a526a58 100644 --- a/filedistribution_test/pom.xml +++ b/filedistribution_test/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>filedistribution_test</artifactId> <version>6-SNAPSHOT</version> diff --git a/filedistributionmanager/pom.xml b/filedistributionmanager/pom.xml index e58112568b0..61de6d2f5a4 100644 --- a/filedistributionmanager/pom.xml +++ b/filedistributionmanager/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>filedistributionmanager</artifactId> <version>6-SNAPSHOT</version> diff --git a/fnet/.gitignore b/fnet/.gitignore index 1ec479be9bc..b5f5f1915cd 100644 --- a/fnet/.gitignore +++ b/fnet/.gitignore @@ -7,4 +7,3 @@ include lib update.log Makefile -Testing diff --git a/frtstream/.gitignore b/frtstream/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/frtstream/.gitignore +++ b/frtstream/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/fsa/.gitignore b/fsa/.gitignore index be0452bed21..4b160dd6a2e 100644 --- a/fsa/.gitignore +++ b/fsa/.gitignore @@ -1,4 +1,3 @@ /target /pom.xml.build Makefile -Testing diff --git a/fsa/pom.xml b/fsa/pom.xml index aef9682deaa..e173a836864 100644 --- a/fsa/pom.xml +++ b/fsa/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fsa</artifactId> <packaging>container-plugin</packaging> diff --git a/functions.cmake b/functions.cmake index 46b8083b94c..8bd0e3a97d1 100644 --- a/functions.cmake +++ b/functions.cmake @@ -151,7 +151,7 @@ endfunction() function(vespa_add_library TARGET) cmake_parse_arguments(ARG - "STATIC;OBJECT;INTERFACE" + "STATIC;OBJECT;INTERFACE;TEST" "INSTALL;OUTPUT_NAME" "DEPENDS;AFTER;SOURCES" ${ARGN}) @@ -178,6 +178,8 @@ function(vespa_add_library TARGET) add_library(${TARGET} ${LINKAGE} ${LIBRARY_TYPE} ${SOURCE_FILES}) __add_dependencies_to_target() + __handle_test_targets() + if(ARG_INSTALL) install(TARGETS ${TARGET} DESTINATION ${ARG_INSTALL}) endif() @@ -201,15 +203,7 @@ function(vespa_add_executable TARGET) add_executable(${TARGET} ${ARG_SOURCES}) __add_dependencies_to_target() - # If this is a test executable, add it to the test target for this module - # If building of unit tests is not specified, exclude this target from the all target - if(ARG_TEST) - __add_test_target_to_module(${TARGET}) - - if(EXCLUDE_TESTS_FROM_ALL) - set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL TRUE) - endif() - endif() + __handle_test_targets() if(ARG_INSTALL) install(TARGETS ${TARGET} DESTINATION ${ARG_INSTALL}) @@ -439,6 +433,18 @@ function(__add_test_target_to_module TARGET) set_property(GLOBAL APPEND PROPERTY MODULE_${MODULE_NAME}_TEST_TARGETS ${TARGET}) endfunction() +macro(__handle_test_targets) + # If this is a test executable, add it to the test target for this module + # If building of unit tests is not specified, exclude this target from the all target + if(ARG_TEST) + __add_test_target_to_module(${TARGET}) + + if(EXCLUDE_TESTS_FROM_ALL) + set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL TRUE) + endif() + endif() +endmacro() + function(__create_module_targets PROPERTY_POSTFIX TARGET_POSTFIX) get_property(VESPA_MODULES GLOBAL PROPERTY VESPA_MODULES) set(OUTPUT_ALL_TARGET "all_${TARGET_POSTFIX}s") diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index a4328091d35..d163ad5fdbd 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>indexinglanguage</artifactId> <packaging>jar</packaging> diff --git a/install_java.cmake b/install_java.cmake index 567cafd881c..a779fcf51d4 100644 --- a/install_java.cmake +++ b/install_java.cmake @@ -2,33 +2,75 @@ function(install_java_artifact NAME) install(FILES "${NAME}/target/${NAME}.jar" DESTINATION lib/jars/) endfunction() +function(install_java_artifact_dependencies NAME) + install(DIRECTORY "${NAME}/target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") +endfunction() + function(install_fat_java_artifact NAME) install(FILES "${NAME}/target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) endfunction() +install_java_artifact(config-model-fat) install_java_artifact(document) -install_java_artifact(searchlib) +install_java_artifact(jdisc_jetty) +install_java_artifact_dependencies(jdisc_jetty) +install_java_artifact_dependencies(vespa_jersey2) install_java_artifact(vespajlib) install_fat_java_artifact(application-preprocessor) +install_fat_java_artifact(clustercontroller-apps) +install_fat_java_artifact(clustercontroller-apputil) +install_fat_java_artifact(clustercontroller-utils) +install_fat_java_artifact(clustercontroller-core) install_fat_java_artifact(component) install_fat_java_artifact(config-bundle) +install_fat_java_artifact(config-model-api) +install_fat_java_artifact(config-model) +install_fat_java_artifact(config-provisioning) +install_fat_java_artifact(config-proxy) install_fat_java_artifact(configdefinitions) +install_fat_java_artifact(configserver) install_fat_java_artifact(container-disc) install_fat_java_artifact(container-jersey2) install_fat_java_artifact(container-search-and-docproc) +install_fat_java_artifact(defaults) install_fat_java_artifact(docprocs) install_fat_java_artifact(jdisc_core) install_fat_java_artifact(jdisc_http_service) +install_fat_java_artifact(logserver) +install_fat_java_artifact(node-repository) +install_fat_java_artifact(orchestrator) install_fat_java_artifact(persistence) +install_fat_java_artifact(searchlib) install_fat_java_artifact(simplemetrics) install_fat_java_artifact(standalone-container) +install_fat_java_artifact(vespa-http-client) install_fat_java_artifact(vespaclient-container-plugin) +install_fat_java_artifact(zkfacade) +vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin) vespa_install_script(jdisc_core/src/main/perl/jdisc_logfmt bin) +vespa_install_script(config-proxy/src/main/sh/config-ctl bin) +vespa_install_script(config-model/src/main/perl/deploy bin) +vespa_install_script(config-model/src/main/perl/deploy-application bin) +vespa_install_script(config-model/src/main/perl/expand-config.pl bin) +vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin) +vespa_install_script(config-model/src/main/sh/validate-application bin) +vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) + +vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin) + +install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) +install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) + install(FILES jdisc_core/src/main/perl/jdisc_logfmt.1 DESTINATION man/man1) install(FILES + config-model-fat/src/main/resources/config-models.xml + node-repository/src/main/config/node-repository.xml + DESTINATION conf/configserver-app) + +install(FILES chain/src/main/resources/configdefinitions/chains.def container-accesslogging/src/main/resources/configdefinitions/access-log.def container-core/src/main/resources/configdefinitions/application-metadata.def diff --git a/jaxrs_client_utils/OWNERS b/jaxrs_client_utils/OWNERS index 9ecc8472a21..f44aaadea3c 100644 --- a/jaxrs_client_utils/OWNERS +++ b/jaxrs_client_utils/OWNERS @@ -1,2 +1,2 @@ bakksjo -hakon +hakonhall diff --git a/jaxrs_client_utils/pom.xml b/jaxrs_client_utils/pom.xml index 1c97bb7fdcc..f449b8f1fb8 100644 --- a/jaxrs_client_utils/pom.xml +++ b/jaxrs_client_utils/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jaxrs_client_utils</artifactId> <version>6-SNAPSHOT</version> diff --git a/jaxrs_utils/OWNERS b/jaxrs_utils/OWNERS index 9ecc8472a21..f44aaadea3c 100644 --- a/jaxrs_utils/OWNERS +++ b/jaxrs_utils/OWNERS @@ -1,2 +1,2 @@ bakksjo -hakon +hakonhall diff --git a/jaxrs_utils/pom.xml b/jaxrs_utils/pom.xml index 3f23c142e62..5dd5c9a22c1 100644 --- a/jaxrs_utils/pom.xml +++ b/jaxrs_utils/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jaxrs_utils</artifactId> <version>6-SNAPSHOT</version> diff --git a/jdisc_akamai/OWNERS b/jdisc_akamai/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_akamai/OWNERS +++ b/jdisc_akamai/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_akamai/src/main/perl/jdisc_akamai_conf.pl b/jdisc_akamai/src/main/perl/jdisc_akamai_conf.pl index 13a0db27552..10ee47cfa50 100644..100755 --- a/jdisc_akamai/src/main/perl/jdisc_akamai_conf.pl +++ b/jdisc_akamai/src/main/perl/jdisc_akamai_conf.pl @@ -1,9 +1,10 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -62,6 +63,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/jdisc_container_maven_archetype_application/OWNERS b/jdisc_container_maven_archetype_application/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_container_maven_archetype_application/OWNERS +++ b/jdisc_container_maven_archetype_application/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_container_maven_archetype_application/pom.xml b/jdisc_container_maven_archetype_application/pom.xml index cfa57ae5857..a7073ded403 100644 --- a/jdisc_container_maven_archetype_application/pom.xml +++ b/jdisc_container_maven_archetype_application/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <!-- Package name must contain _maven_archetype --> <artifactId>jdisc_container_maven_archetype_application</artifactId> diff --git a/jdisc_core/OWNERS b/jdisc_core/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core/OWNERS +++ b/jdisc_core/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 7bd1a54d330..4704c417230 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_core</artifactId> <version>6-SNAPSHOT</version> @@ -121,13 +120,7 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <!-- Used for export-package parsing. Ideally, that should have been a separate artifact. --> + <!-- This seems odd. Used for export-package parsing. Lazy stuff. Should be separated out. --> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> @@ -212,7 +205,6 @@ <argument>${project.build.directory}/dependency/log4j-over-slf4j.jar</argument> <argument>${project.build.directory}/dependency/config-lib.jar</argument> <argument>${project.build.directory}/dependency/yolean.jar</argument> - <argument>${project.build.directory}/dependency/vespajlib.jar</argument> </arguments> </configuration> </execution> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index 552d423f8cb..19b6c220f07 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.yahoo.system.CatchSigTerm; +import com.yahoo.yolean.system.CatchSigTerm; import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,16 +36,17 @@ public class StandaloneMain { void run(String bundleLocation) { try { - log.info("Initializing application without privileges."); - loader.init(bundleLocation, false); - loader.start(); - setupSigTermHandler(); - waitForShutdown(); - // Event.stopping(APPNAME, "shutdown"); - loader.stop(); - // Event.stopped(APPNAME, 0, 0); - loader.destroy(); - // System.exit(0); + System.out.println("debug\tInitializing application without privileges."); + loader.init(bundleLocation, false); + loader.start(); + setupSigTermHandler(); + waitForShutdown(); + System.out.println("debug\tTrying to shutdown in a controlled manner."); + loader.stop(); + System.out.println("debug\tTrying to clean up in a controlled manner."); + loader.destroy(); + System.out.println("debug\tStopped ok."); + System.exit(0); } catch (Exception e) { log.log(Level.WARNING, "Unexpected: ", e); System.exit(6); diff --git a/jdisc_core/src/main/perl/jdisc_logfmt b/jdisc_core/src/main/perl/jdisc_logfmt index 1a05e229832..11a21cbe01a 100755 --- a/jdisc_core/src/main/perl/jdisc_logfmt +++ b/jdisc_core/src/main/perl/jdisc_logfmt @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section diff --git a/jdisc_core/src/test/perl/jdisc_logfmt_test.sh b/jdisc_core/src/test/perl/jdisc_logfmt_test.sh index bb7e92ed8cf..0166d33d8ab 100755 --- a/jdisc_core/src/test/perl/jdisc_logfmt_test.sh +++ b/jdisc_core/src/test/perl/jdisc_logfmt_test.sh @@ -4,10 +4,10 @@ MYPATH=`dirname ${0}` DIFF=/usr/bin/diff LOGFMT=${1} -if [ -e "/usr/local/bin/perl" ]; then +if which perl &> /dev/null; then echo "Running jdisc_logfmt test suite." else - echo "Ignoring jdisc_logfmt test suite as there is no /usr/local/bin/perl" + echo "Ignoring jdisc_logfmt test suite as there is no perl executable." exit 0 fi diff --git a/jdisc_core_test/OWNERS b/jdisc_core_test/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core_test/OWNERS +++ b/jdisc_core_test/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_core_test/pom.xml b/jdisc_core_test/pom.xml index 85b4d7c1c37..360d97ff7a7 100644 --- a/jdisc_core_test/pom.xml +++ b/jdisc_core_test/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>parent</artifactId> diff --git a/jdisc_http_service/OWNERS b/jdisc_http_service/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_http_service/OWNERS +++ b/jdisc_http_service/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 15518d20c62..fdae8728ea9 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_http_service</artifactId> <version>6-SNAPSHOT</version> diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index e9aba0cb6c9..c16ac589332 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -101,6 +101,13 @@ class HttpRequestDispatch { HttpRequestDispatch parent = this; //used to avoid binding uninitialized variables completeRequestCallback = (result, error) -> { + boolean alreadyCalled = completeRequestCalled.getAndSet(true); + if (alreadyCalled) { + AssertionError e = new AssertionError("completeRequest called more than once"); + log.log(Level.WARNING, "Assertion failed.", e); + throw e; + } + boolean reportedError = false; if (error != null) { @@ -113,14 +120,6 @@ class HttpRequestDispatch { parent.metricReporter.successfulResponse(); } - - boolean alreadyCalled = completeRequestCalled.getAndSet(true); - if (alreadyCalled) { - AssertionError e = new AssertionError("completeRequest called more than once"); - log.log(Level.WARNING, "Assertion failed.", e); - throw e; - } - try { parent.async.complete(); log.finest(() -> "Request completed successfully: " + parent.servletRequest.getRequestURI()); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index 271805765c2..7a15107a3eb 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -41,6 +41,10 @@ public class ServletOutputStreamWriter { private static final Logger log = Logger.getLogger(ServletOutputStreamWriter.class.getName()); + // TODO: This reference is not guaranteed to be unique; ByteBuffer.allocate(0) MAY in principle return a singleton! + // If so, application code could fake a close by writing such a byte buffer. + // The problem can be solved by filtering out zero-length byte buffers from application code. + // Other ways to express this are also possible, e.g. with a 'closed' state checked when queue goes empty. private static final ByteBuffer CLOSE_STREAM_BUFFER = ByteBuffer.allocate(0); private final Object monitor = new Object(); @@ -74,6 +78,7 @@ public class ServletOutputStreamWriter { public void setSendingError() { synchronized (monitor) { + // TODO: This assert seems fishy. Investigate. assertStateIs(state, State.NOT_STARTED); state = State.FINISHED_OR_ERROR; } @@ -109,6 +114,7 @@ public class ServletOutputStreamWriter { } if (thisThreadShouldWrite) { + // TODO: Consider refactoring to avoid multiple monitor entry-exit. writeBuffersInQueueToOutputStream(); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 5bea01bd104..a763a03d39d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -133,9 +133,8 @@ class ServletRequestReader implements ReadListener { numberOfOutstandingUserCalls += 2; } try { - requestContentChannel.write(buf, writeCompletionHandler); - int bytesReceived = buf.remaining(); + requestContentChannel.write(buf, writeCompletionHandler); metricReporter.successfulRead(bytesReceived); } catch (final Throwable t) { finishedFuture.completeExceptionally(t); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index b0781c402d5..126e4fee9e6 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -115,6 +115,8 @@ public class ServletResponseController { } try { + // TODO: sendError() is a synchronous call. Refactor. (Also, we should control the response content - + // this method generates a response body based on Jetty's own response templates ("Powered by Jetty"). servletResponse.sendError( statusCode, reasonPhrase); @@ -166,10 +168,14 @@ public class ServletResponseController { private static void setStatus_holdingLock(Response jdiscResponse, HttpServletResponse servletResponse) { if (jdiscResponse instanceof HttpResponse) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), ((HttpResponse) jdiscResponse).getMessage()); } else { Optional<String> errorMessage = getErrorMessage(jdiscResponse); if (errorMessage.isPresent()) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), errorMessage.get()); } else { servletResponse.setStatus(jdiscResponse.getStatus()); diff --git a/jdisc_jetty/OWNERS b/jdisc_jetty/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_jetty/OWNERS +++ b/jdisc_jetty/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_jetty/pom.xml b/jdisc_jetty/pom.xml index 90aae3b0d13..b3e147bdac5 100644 --- a/jdisc_jetty/pom.xml +++ b/jdisc_jetty/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_jetty</artifactId> <version>6-SNAPSHOT</version> diff --git a/jdisc_jmx_metrics/OWNERS b/jdisc_jmx_metrics/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_jmx_metrics/OWNERS +++ b/jdisc_jmx_metrics/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_jmx_metrics/pom.xml b/jdisc_jmx_metrics/pom.xml index 75970944d7e..bc9b52adb80 100644 --- a/jdisc_jmx_metrics/pom.xml +++ b/jdisc_jmx_metrics/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_jmx_metrics</artifactId> <version>6-SNAPSHOT</version> diff --git a/jdisc_maven_archetype_component/OWNERS b/jdisc_maven_archetype_component/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_maven_archetype_component/OWNERS +++ b/jdisc_maven_archetype_component/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_maven_archetype_component/pom.xml b/jdisc_maven_archetype_component/pom.xml index 6e4d5221859..2926eccabcb 100644 --- a/jdisc_maven_archetype_component/pom.xml +++ b/jdisc_maven_archetype_component/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <!-- Package name must contain _maven_archetype --> <artifactId>jdisc_maven_archetype_component</artifactId> diff --git a/jdisc_messagebus_service/OWNERS b/jdisc_messagebus_service/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_messagebus_service/OWNERS +++ b/jdisc_messagebus_service/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_messagebus_service/pom.xml b/jdisc_messagebus_service/pom.xml index 62e29e4bc5d..cf844e50a90 100644 --- a/jdisc_messagebus_service/pom.xml +++ b/jdisc_messagebus_service/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_messagebus_service</artifactId> <version>6-SNAPSHOT</version> diff --git a/jdisc_status/OWNERS b/jdisc_status/OWNERS index d24d7c7860d..f02be9b1f86 100644 --- a/jdisc_status/OWNERS +++ b/jdisc_status/OWNERS @@ -1,2 +1,3 @@ gjoranv bakksjo +bjorncs diff --git a/jdisc_status/src/main/perl/jdisc_status_conf.pl b/jdisc_status/src/main/perl/jdisc_status_conf.pl index 3e02f42c46d..bd635e1cf07 100644..100755 --- a/jdisc_status/src/main/perl/jdisc_status_conf.pl +++ b/jdisc_status/src/main/perl/jdisc_status_conf.pl @@ -1,9 +1,10 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -62,6 +63,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/jrt/pom.xml b/jrt/pom.xml index 967d75e8937..6592cd84638 100644 --- a/jrt/pom.xml +++ b/jrt/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jrt</artifactId> <packaging>container-plugin</packaging> diff --git a/jrt_test/.gitignore b/jrt_test/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/jrt_test/.gitignore +++ b/jrt_test/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/jrt_test/src/java/.gitignore b/jrt_test/src/java/.gitignore index c7d02fa8c22..ce75892238c 100644 --- a/jrt_test/src/java/.gitignore +++ b/jrt_test/src/java/.gitignore @@ -1,3 +1,4 @@ build.inc classes jrt-test.jar +/java_code_compiled diff --git a/jrt_test/src/tests/echo/echo_test.sh b/jrt_test/src/tests/echo/echo_test.sh index 49241e787f5..df6d98eaaf1 100755 --- a/jrt_test/src/tests/echo/echo_test.sh +++ b/jrt_test/src/tests/echo/echo_test.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e . ../../binref/env.sh sh dotest.sh || (sh $BINREF/progctl.sh progdefs.sh stop all; false) sh $BINREF/progctl.sh progdefs.sh stop all diff --git a/jrt_test/src/tests/mandatory-methods/mandatory-methods_test.sh b/jrt_test/src/tests/mandatory-methods/mandatory-methods_test.sh index bf86849d9c1..3088682ef91 100755 --- a/jrt_test/src/tests/mandatory-methods/mandatory-methods_test.sh +++ b/jrt_test/src/tests/mandatory-methods/mandatory-methods_test.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e . ../../binref/env.sh export PORT_1 diff --git a/jrt_test/src/tests/mockup-invoke/mockup-invoke_test.sh b/jrt_test/src/tests/mockup-invoke/mockup-invoke_test.sh index 56ca7079e7a..d5ac8535393 100755 --- a/jrt_test/src/tests/mockup-invoke/mockup-invoke_test.sh +++ b/jrt_test/src/tests/mockup-invoke/mockup-invoke_test.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e . ../../binref/env.sh diff --git a/jrt_test/src/tests/rpc-error/rpc-error_test.sh b/jrt_test/src/tests/rpc-error/rpc-error_test.sh index dc2f855acbd..a5ef9ac5b3e 100755 --- a/jrt_test/src/tests/rpc-error/rpc-error_test.sh +++ b/jrt_test/src/tests/rpc-error/rpc-error_test.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e . ../../binref/env.sh diff --git a/juniper/.gitignore b/juniper/.gitignore index 92af948a4ce..0c9cc229d0e 100644 --- a/juniper/.gitignore +++ b/juniper/.gitignore @@ -6,4 +6,3 @@ juniper.mak lib project.dsw Makefile -Testing diff --git a/linguistics/pom.xml b/linguistics/pom.xml index baeb2457c76..3c8ae341249 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>linguistics</artifactId> <packaging>container-plugin</packaging> diff --git a/logd/.gitignore b/logd/.gitignore index fd963ca589b..91220ac384b 100644 --- a/logd/.gitignore +++ b/logd/.gitignore @@ -3,4 +3,3 @@ conf /target /pom.xml.build Makefile -Testing diff --git a/logd/pom.xml b/logd/pom.xml index 8385960066e..b8d49e21677 100644 --- a/logd/pom.xml +++ b/logd/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logd</artifactId> <version>6-SNAPSHOT</version> diff --git a/logserver/pom.xml b/logserver/pom.xml index dc997db0386..c982a442cf5 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logserver</artifactId> <packaging>jar</packaging> diff --git a/lowercasing_test/.gitignore b/lowercasing_test/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/lowercasing_test/.gitignore +++ b/lowercasing_test/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/memfilepersistence/.gitignore b/memfilepersistence/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/memfilepersistence/.gitignore +++ b/memfilepersistence/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/messagebus-disc/OWNERS b/messagebus-disc/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/messagebus-disc/OWNERS +++ b/messagebus-disc/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/messagebus-disc/pom.xml b/messagebus-disc/pom.xml index 60393f48ebd..5357de9c8ed 100644 --- a/messagebus-disc/pom.xml +++ b/messagebus-disc/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>messagebus-disc</artifactId> <version>6-SNAPSHOT</version> diff --git a/messagebus/.gitignore b/messagebus/.gitignore index b79b059b0a7..a5c2cde721d 100644 --- a/messagebus/.gitignore +++ b/messagebus/.gitignore @@ -4,4 +4,3 @@ messagebus-lib.iml target /pom.xml.build Makefile -Testing diff --git a/messagebus/CMakeLists.txt b/messagebus/CMakeLists.txt index 06a37a0ec95..534887e011b 100644 --- a/messagebus/CMakeLists.txt +++ b/messagebus/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_define_module( vespalib staging_vespalib fnet - slobrok slobrok_slobrokserver LIBS diff --git a/messagebus/OWNERS b/messagebus/OWNERS index b9557349aa2..7379f3287ad 100644 --- a/messagebus/OWNERS +++ b/messagebus/OWNERS @@ -1,2 +1,3 @@ -dybdahl +dybis baldersheim + diff --git a/messagebus/pom.xml b/messagebus/pom.xml index 16868d49e7c..a3ae2fb54c2 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>messagebus</artifactId> <version>6-SNAPSHOT</version> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java index 72f13ac2a1e..469deefa7f7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus; /** * This is an implementation of the {@link ThrottlePolicy} that passes all requests (no real throttling). - * @author <a href="mailto:dybdahl@yahoo-inc.com">Haakon Dybdahl</a> + * @author dybis */ public class AllPassThrottlePolicy implements ThrottlePolicy { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java b/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java index 52b3d824019..4753fc1db1f 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java @@ -30,12 +30,7 @@ public class Identity { * @param configId The config identifier for the application. */ public Identity(String configId) { - InetAddress addr; - try { - addr = LinuxInetAddress.getLocalHost(); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } + InetAddress addr = LinuxInetAddress.getLocalHost(); if (addr instanceof Inet6Address) { log.log(LogLevel.WARNING, "Local host resolved to IPv6 address '" + addr.getHostAddress() + "', this might be problematic."); diff --git a/metrics/.gitignore b/metrics/.gitignore index cb87b051cf2..a9b9304101f 100644 --- a/metrics/.gitignore +++ b/metrics/.gitignore @@ -2,4 +2,3 @@ /target /pom.xml.build Makefile -Testing diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 03da6f2ee64..80b6ef64c7b 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -7,10 +7,12 @@ vespa_define_module( staging_vespalib config_cloudconfig + LIBS + src/vespa/metrics + TEST_EXTERNAL_DEPENDS cppunit TESTS src/tests - src/vespa/metrics ) diff --git a/metrics/pom.xml b/metrics/pom.xml index 0aa81dff64b..af78b9e7c91 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>metrics</artifactId> diff --git a/node-admin/OWNERS b/node-admin/OWNERS index 9ecc8472a21..f44aaadea3c 100644 --- a/node-admin/OWNERS +++ b/node-admin/OWNERS @@ -1,2 +1,2 @@ bakksjo -hakon +hakonhall diff --git a/node-admin/pom.xml b/node-admin/pom.xml index c89e8524fc3..6c62d3c7f46 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-admin</artifactId> diff --git a/node-admin/scripts/node-admin.sh b/node-admin/scripts/node-admin.sh index ae0fa94029b..cae3cf54d0d 100755 --- a/node-admin/scripts/node-admin.sh +++ b/node-admin/scripts/node-admin.sh @@ -123,7 +123,6 @@ function Start { --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/vespa:$VESPA_HOME/var/vespa" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/yca:$VESPA_HOME/var/yca" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ycore++:$VESPA_HOME/var/ycore++" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ymon:$VESPA_HOME/var/ymon" \ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/zookeeper:$VESPA_HOME/var/zookeeper" \ --env "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_HOSTNAME" \ --env "NETWORK_TYPE=$NETWORK_TYPE" \ diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index f2b31b3afb9..fa7307fc5eb 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -1,13 +1,13 @@ <?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> <jdisc version="1.0" jetty="true"> - <rest-api path="test" jersey2="true"> - <components bundle="node-admin"> - <package>com.yahoo.vespa.hosted.node.admin.testapi</package> - </components> - </rest-api> - <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.NodeAdminScheduler" bundle="node-admin"/> + <!-- Please update container test when changing this file --> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> + <component id="docker" class="com.yahoo.vespa.hosted.node.admin.docker.DockerImpl" bundle="node-admin"/> + <config name='nodeadmin.docker.docker'> <caCertPath>/host/docker/certs/ca_cert.pem</caCertPath> <clientCertPath>/host/docker/certs/client_cert.pem</clientCertPath> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java index 6d4873d92bf..863a8f47965 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java @@ -1,155 +1,23 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin; -import com.yahoo.collections.Pair; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.node.admin.docker.Container; -import com.yahoo.vespa.hosted.node.admin.docker.Docker; -import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** - * The "most important" class in this module, where the main business logic resides or is driven from. - * - * @author stiankri + * API for NodeAdmin seen from outside. + * @author dybis */ -public class NodeAdmin { - private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName()); +public interface NodeAdmin { - private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis(); + void refreshContainersToRun(final List<ContainerNodeSpec> containersToRun); - private final Docker docker; - private final Function<HostName, NodeAgent> nodeAgentFactory; + boolean freezeAndCheckIfAllFrozen(); - private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>(); + void unfreeze(); - private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap(); + Set<HostName> getListOfHosts(); - /** - * @param docker interface to docker daemon and docker-related tasks - * @param nodeAgentFactory factory for {@link NodeAgent} objects - */ - public NodeAdmin(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) { - this.docker = docker; - this.nodeAgentFactory = nodeAgentFactory; - } - - public void maintainWantedState(final List<ContainerNodeSpec> containersToRun) { - final List<Container> existingContainers = docker.getAllManagedContainers(); - - synchronizeLocalContainerState(containersToRun, existingContainers); - - garbageCollectDockerImages(containersToRun); - } - - private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) { - final Set<DockerImage> deletableDockerImages = getDeletableDockerImages( - docker.getUnusedDockerImages(), containersToRun); - final long currentTime = System.currentTimeMillis(); - // TODO: This logic should be unit tested. - firstTimeEligibleForGC = deletableDockerImages.stream() - .collect(Collectors.toMap( - dockerImage -> dockerImage, - dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime))); - // Delete images that have been eligible for some time. - firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> { - if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) { - docker.deleteImage(dockerImage); - } - }); - } - - // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present. - // This is a workaround for Java 8 not having Stream.flatMap(Optional). - private static <T> Stream<T> streamOf(Optional<T> opt) { - return opt.map(Stream::of) - .orElseGet(Stream::empty); - } - - static Set<DockerImage> getDeletableDockerImages( - final Set<DockerImage> currentlyUnusedDockerImages, - final List<ContainerNodeSpec> pendingContainers) { - final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream() - .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage)) - .collect(Collectors.toSet()); - return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture); - } - - // Set-difference. Returns minuend minus subtrahend. - private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) { - final HashSet<T> result = new HashSet<>(minuend); - result.removeAll(subtrahend); - return result; - } - - // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V). - // Full outer join means that all elements of both data sources are included in the result, - // even when there is no corresponding element (having the same attribute) in the other data set, - // in which case the value from the other source will be empty. - static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin( - final Stream<T> tStream, final Function<T, V> tAttributeExtractor, - final Stream<U> uStream, final Function<U, V> uAttributeExtractor) { - final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t)); - final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u)); - return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream()) - .distinct() - .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key)))); - } - - void synchronizeLocalContainerState( - final List<ContainerNodeSpec> containersToRun, - final List<Container> existingContainers) { - final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin( - containersToRun.stream(), nodeSpec -> nodeSpec.hostname, - existingContainers.stream(), container -> container.hostname); - - final Set<HostName> nodeHostNames = containersToRun.stream() - .map(spec -> spec.hostname) - .collect(Collectors.toSet()); - final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames); - obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).stop()); - - nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> { - final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst(); - final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond(); - - if (!nodeSpec.isPresent()) { - assert existingContainer.isPresent(); - logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist"); - return; - } - - try { - updateAgent(nodeSpec.get()); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to bring container to desired state", e); - } - }); - } - - private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException { - final NodeAgent agent; - if (nodeAgents.containsKey(nodeSpec.hostname)) { - agent = nodeAgents.get(nodeSpec.hostname); - } else { - agent = nodeAgentFactory.apply(nodeSpec.hostname); - nodeAgents.put(nodeSpec.hostname, agent); - agent.start(); - } - agent.update(); - } + String debugInfo(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java new file mode 100644 index 00000000000..f9e131e3d6c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java @@ -0,0 +1,187 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.docker.Container; +import com.yahoo.vespa.hosted.node.admin.docker.Docker; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Administers a host (for now only docker hosts) and its nodes (docker containers nodes). + * + * @author stiankri + */ +public class NodeAdminImpl implements NodeAdmin { + private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName()); + + private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis(); + + private final Docker docker; + private final Function<HostName, NodeAgent> nodeAgentFactory; + + private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>(); + + private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap(); + + /** + * @param docker interface to docker daemon and docker-related tasks + * @param nodeAgentFactory factory for {@link NodeAgent} objects + */ + public NodeAdminImpl(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) { + this.docker = docker; + this.nodeAgentFactory = nodeAgentFactory; + } + + public void refreshContainersToRun(final List<ContainerNodeSpec> containersToRun) { + final List<Container> existingContainers = docker.getAllManagedContainers(); + + synchronizeLocalContainerState(containersToRun, existingContainers); + + garbageCollectDockerImages(containersToRun); + } + + public boolean freezeAndCheckIfAllFrozen() { + for (NodeAgent nodeAgent : nodeAgents.values()) { + nodeAgent.execute(NodeAgent.Command.FREEZE); + } + for (NodeAgent nodeAgent : nodeAgents.values()) { + if (nodeAgent.getState() != NodeAgent.State.FROZEN) { + return false; + } + } + return true; + } + + public void unfreeze() { + for (NodeAgent nodeAgent : nodeAgents.values()) { + nodeAgent.execute(NodeAgent.Command.UNFREEZE); + } + } + + public Set<HostName> getListOfHosts() { + return nodeAgents.keySet(); + } + + @Override + public String debugInfo() { + StringBuilder debug = new StringBuilder(); + for (Map.Entry<HostName, NodeAgent> node : nodeAgents.entrySet()) { + debug.append("Node ").append(node.getKey().toString()); + debug.append(" state ").append(node.getValue().getState()); + } + return debug.toString(); + } + + private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) { + final Set<DockerImage> deletableDockerImages = getDeletableDockerImages( + docker.getUnusedDockerImages(), containersToRun); + final long currentTime = System.currentTimeMillis(); + // TODO: This logic should be unit tested. + firstTimeEligibleForGC = deletableDockerImages.stream() + .collect(Collectors.toMap( + dockerImage -> dockerImage, + dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime))); + // Delete images that have been eligible for some time. + firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> { + if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) { + docker.deleteImage(dockerImage); + } + }); + } + + // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present. + // This is a workaround for Java 8 not having Stream.flatMap(Optional). + private static <T> Stream<T> streamOf(Optional<T> opt) { + return opt.map(Stream::of) + .orElseGet(Stream::empty); + } + + static Set<DockerImage> getDeletableDockerImages( + final Set<DockerImage> currentlyUnusedDockerImages, + final List<ContainerNodeSpec> pendingContainers) { + final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream() + .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage)) + .collect(Collectors.toSet()); + return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture); + } + + // Set-difference. Returns minuend minus subtrahend. + private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) { + final HashSet<T> result = new HashSet<>(minuend); + result.removeAll(subtrahend); + return result; + } + + // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V). + // Full outer join means that all elements of both data sources are included in the result, + // even when there is no corresponding element (having the same attribute) in the other data set, + // in which case the value from the other source will be empty. + static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin( + final Stream<T> tStream, final Function<T, V> tAttributeExtractor, + final Stream<U> uStream, final Function<U, V> uAttributeExtractor) { + final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t)); + final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u)); + return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream()) + .distinct() + .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key)))); + } + + void synchronizeLocalContainerState( + final List<ContainerNodeSpec> containersToRun, + final List<Container> existingContainers) { + final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin( + containersToRun.stream(), nodeSpec -> nodeSpec.hostname, + existingContainers.stream(), container -> container.hostname); + + final Set<HostName> nodeHostNames = containersToRun.stream() + .map(spec -> spec.hostname) + .collect(Collectors.toSet()); + final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames); + obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).stop()); + + nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> { + final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst(); + final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond(); + + if (!nodeSpec.isPresent()) { + assert existingContainer.isPresent(); + logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist"); + return; + } + + try { + updateAgent(nodeSpec.get()); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to bring container to desired state", e); + } + }); + } + + private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException { + final NodeAgent agent; + if (nodeAgents.containsKey(nodeSpec.hostname)) { + agent = nodeAgents.get(nodeSpec.hostname); + } else { + agent = nodeAgentFactory.apply(nodeSpec.hostname); + nodeAgents.put(nodeSpec.hostname, agent); + agent.start(); + } + agent.execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java deleted file mode 100644 index 985e20a3ea8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import com.yahoo.nodeadmin.docker.DockerConfig; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.node.admin.docker.Docker; -import com.yahoo.vespa.hosted.node.admin.docker.DockerImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; - -import javax.annotation.concurrent.GuardedBy; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static java.util.concurrent.TimeUnit.SECONDS; - -/** - * @author stiankri - */ -public class NodeAdminScheduler extends AbstractComponent { - private static final Logger log = Logger.getLogger(NodeAdminScheduler.class.getName()); - - private static final long INITIAL_DELAY_SECONDS = 0; - private static final long INTERVAL_IN_SECONDS = 60; - - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - private final ScheduledFuture<?> scheduledFuture; - - private enum State { WAIT, WORK, STOP } - - private final Object monitor = new Object(); - @GuardedBy("monitor") - private State state = State.WAIT; - @GuardedBy("monitor") - private List<ContainerNodeSpec> wantedContainerState = null; - - public NodeAdminScheduler(final DockerConfig dockerConfig) { - final Docker docker = new DockerImpl(DockerImpl.newDockerClientFromConfig(dockerConfig)); - final NodeRepository nodeRepository = new NodeRepositoryImpl(); - final Orchestrator orchestrator = new OrchestratorImpl(OrchestratorImpl.makeOrchestratorHostApiClient()); - final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> - new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator); - final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory); - scheduledFuture = scheduler.scheduleWithFixedDelay( - throwableLoggingRunnable(fetchContainersToRunFromNodeRepository(nodeRepository)), - INITIAL_DELAY_SECONDS, INTERVAL_IN_SECONDS, SECONDS); - new Thread(maintainWantedStateRunnable(nodeAdmin), "Node Admin Scheduler main thread").start(); - } - - private void notifyWorkToDo(final Runnable codeToExecuteInCriticalSection) { - synchronized (monitor) { - if (state == State.STOP) { - return; - } - state = State.WORK; - codeToExecuteInCriticalSection.run(); - monitor.notifyAll(); - } - } - - /** - * Prevents exceptions from leaking out and suppressing the scheduler from running the task again. - */ - private static Runnable throwableLoggingRunnable(final Runnable task) { - return () -> { - try { - task.run(); - } catch (Throwable throwable) { - log.log(LogLevel.ERROR, "Unhandled exception leaked out to scheduler.", throwable); - } - }; - } - - private Runnable fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) { - return () -> { - // TODO: should the result from the config server contain both active and inactive? - final List<ContainerNodeSpec> containersToRun; - try { - containersToRun = nodeRepository.getContainersToRun(); - } catch (IOException e) { - log.log(Level.WARNING, "Failed fetching container info from node repository", e); - return; - } - setWantedContainerState(containersToRun); - }; - } - - private void setWantedContainerState(final List<ContainerNodeSpec> wantedContainerState) { - if (wantedContainerState == null) { - throw new IllegalArgumentException("wantedContainerState must not be null"); - } - - final Runnable codeToExecuteInCriticalSection = () -> this.wantedContainerState = wantedContainerState; - notifyWorkToDo(codeToExecuteInCriticalSection); - } - - private Runnable maintainWantedStateRunnable(final NodeAdmin nodeAdmin) { - return () -> { - while (true) { - final List<ContainerNodeSpec> containersToRun; - - synchronized (monitor) { - while (state == State.WAIT) { - try { - monitor.wait(); - } catch (InterruptedException e) { - // Ignore, properly handled by next loop iteration. - } - } - if (state == State.STOP) { - return; - } - assert state == State.WORK; - assert wantedContainerState != null; - containersToRun = wantedContainerState; - state = State.WAIT; - } - - throwableLoggingRunnable(() -> nodeAdmin.maintainWantedState(containersToRun)) - .run(); - } - }; - } - - @Override - public void deconstruct() { - scheduledFuture.cancel(false); - scheduler.shutdown(); - synchronized (monitor) { - state = State.STOP; - monitor.notifyAll(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java new file mode 100644 index 00000000000..deb1f37e00e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater.State.RESUMED; +import static com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater.State.SUSPENDED; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * Pulls information from node repository and forwards containers to run to node admin. + * + * @author dybis, stiankri + */ +public class NodeAdminStateUpdater extends AbstractComponent { + private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + + private final NodeAdmin nodeAdmin; + private boolean isRunningUpdates = true; + private final Object monitor = new Object(); + final Orchestrator orchestrator; + private final String baseHostName; + + public NodeAdminStateUpdater( + final NodeRepository nodeRepository, + final NodeAdmin nodeAdmin, + long initialSchedulerDelayMillis, + long intervalSchedulerInMillis, + Orchestrator orchestrator, + String baseHostName) { + scheduler.scheduleWithFixedDelay( + ()-> fetchContainersToRunFromNodeRepository(nodeRepository), + initialSchedulerDelayMillis, + intervalSchedulerInMillis, + MILLISECONDS); + this.nodeAdmin = nodeAdmin; + this.orchestrator = orchestrator; + this.baseHostName = baseHostName; + } + + public String getDebugPage() { + StringBuilder info = new StringBuilder(); + synchronized (monitor) { + info.append("isRunningUpdates is " + isRunningUpdates+ ". "); + info.append("NodeAdmin: "); + info.append(nodeAdmin.debugInfo()); + } + return info.toString(); + } + + public enum State { RESUMED, SUSPENDED} + + /** + * @return empty on success and failure message on failure. + */ + public Optional<String> setResumeStateAndCheckIfResumed(State wantedState) { + synchronized (monitor) { + isRunningUpdates = wantedState == RESUMED; + + if (wantedState == SUSPENDED) { + if (!nodeAdmin.freezeAndCheckIfAllFrozen()) { + return Optional.of("Not all node agents are frozen."); + } + List<String> hosts = new ArrayList<>(); + nodeAdmin.getListOfHosts().forEach(host -> hosts.add(host.toString())); + return orchestrator.suspend(baseHostName, hosts); + } else { + nodeAdmin.unfreeze(); + // we let the NodeAgent do the resume against the orchestrator. + return Optional.empty(); + } + } + } + + private void fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) { + synchronized (monitor) { + if (! isRunningUpdates) { + log.log(Level.FINE, "Is frozen, skipping"); + return; + } + final List<ContainerNodeSpec> containersToRun; + try { + containersToRun = nodeRepository.getContainersToRun(); + } catch (Throwable t) { + log.log(Level.WARNING, "Failed fetching container info from node repository", t); + return; + } + if (containersToRun == null) { + log.log(Level.WARNING, "Got null from NodeRepo."); + return; + } + try { + nodeAdmin.refreshContainersToRun(containersToRun); + } catch (Throwable t) { + log.log(Level.WARNING, "Failed updating node admin: ", t); + return; + } + } + } + + @Override + public void deconstruct() { + scheduler.shutdown(); + try { + if (! scheduler.awaitTermination(30, TimeUnit.SECONDS)) { + throw new RuntimeException("Did not manage to shutdown scheduler."); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java index 54e7ac3e92f..4d5e19e4bce 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java @@ -2,25 +2,36 @@ package com.yahoo.vespa.hosted.node.admin; /** - * Responsible for management of a single node/container over its lifecycle. + * Responsible for management of a single node over its lifecycle. * May own its own resources, threads etc. Runs independently, but receives signals * on state changes in the environment that may trigger this agent to take actions. * * @author bakksjo */ public interface NodeAgent { + + enum Command {UPDATE_FROM_NODE_REPO, FREEZE, UNFREEZE} + enum State {WAITING, WORKING, FROZEN, TERMINATED} + /** * Signals to the agent that it should update the node specification and container state and maintain wanted state. * - * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after this method call returns. + * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after + * this method call returns. * * It is an error to call this method on an instance after stop() has been called. */ - void update(); + void execute(Command wantedState); + + /** + * Returns the state of the agent. + */ + State getState(); /** * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously - * striving to make the current state equal to the wanted state. The current and wanted state update as part of {@link #update()}. + * striving to make the current state equal to the wanted state. The current and wanted state update as part of + * {@link #execute(Command)}. */ void start(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java index b197b639166..d8defb3f736 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,11 +42,11 @@ public class NodeAgentImpl implements NodeAgent { private final Thread thread; - private enum State { WAIT, WORK, STOP } - private final Object monitor = new Object(); @GuardedBy("monitor") - private State state = State.WAIT; + private State state = State.WAITING; + @GuardedBy("monitor") + private State wantedState = State.WAITING; // The attributes of the last successful noderepo attribute update for this node. Used to avoid redundant calls. // Only used internally by maintenance thread; no synchronization necessary. @@ -55,6 +56,7 @@ public class NodeAgentImpl implements NodeAgent { + /** * @param hostName the hostname of the node managed by this agent * @param docker interface to docker daemon and docker-related tasks @@ -75,18 +77,36 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public void update() { - changeStateAndNotify(() -> { - this.state = State.WORK; - }); + public void execute(Command command) { + synchronized (monitor) { + switch (command) { + case UPDATE_FROM_NODE_REPO: + wantedState = State.WORKING; + break; + case FREEZE: + wantedState = State.FROZEN; + break; + case UNFREEZE: + wantedState = State.WORKING; + break; + } + monitor.notifyAll(); + } + } + + @Override + public State getState() { + synchronized (monitor) { + return state; + } } @Override public void start() { logger.log(LogLevel.INFO, logPrefix + "Scheduling start of NodeAgent"); synchronized (monitor) { - if (state == State.STOP) { - throw new IllegalStateException("Cannot re-start a stopped node agent"); + if (state == State.TERMINATED) { + throw new IllegalStateException("Cannot re-start a stopped (terminated) node agent"); } } thread.start(); @@ -95,12 +115,13 @@ public class NodeAgentImpl implements NodeAgent { @Override public void stop() { logger.log(LogLevel.INFO, logPrefix + "Scheduling stop of NodeAgent"); - changeStateAndNotify(() -> { - if (state == State.STOP) { - throw new IllegalStateException("Cannot stop an already stopped node agent"); + synchronized (monitor) { + if (state == State.TERMINATED) { + throw new IllegalStateException("Cannot stop an already stopped (terminated) node agent"); } - state = State.STOP; - }); + wantedState = State.TERMINATED; + monitor.notifyAll(); + } try { thread.join(); } catch (InterruptedException e) { @@ -360,44 +381,62 @@ public class NodeAgentImpl implements NodeAgent { } private void scheduleWork() { - changeStateAndNotify(() -> state = State.WORK); - } - - private void changeStateAndNotify(final Runnable stateChanger) { synchronized (monitor) { - if (state == State.STOP) { - return; + if (wantedState != State.FROZEN) { + wantedState = State.WORKING; + } else { + logger.log(Level.FINE, "Not scheduling work since in freeze."); } - stateChanger.run(); monitor.notifyAll(); } } - private void maintainWantedState() { - while (true) { - synchronized (monitor) { - while (state == State.WAIT) { - try { + private void blockUntilNotWaitingOrFrozen() { + try { + synchronized (monitor) { + while (wantedState == State.WAITING || wantedState == State.FROZEN) { + state = wantedState; monitor.wait(); - } catch (InterruptedException e) { - // Ignore, properly handled by next loop iteration. + continue; } } - if (state == State.STOP) { - return; - } - assert state == State.WORK; - state = State.WAIT; + } catch (InterruptedException e) { + logger.severe("NodeAgent thread interrupted. Ignoring this: " + e.getMessage()); } + } + private void maintainWantedState() { + while (true) { + blockUntilNotWaitingOrFrozen(); + synchronized (monitor) { + switch (wantedState) { + case WAITING: + state = State.WAITING; + continue; + case WORKING: + state = State.WORKING; + wantedState = State.WAITING; + break; + case FROZEN: + state = State.FROZEN; + continue; + case TERMINATED: + return; + } + } + // state is WORKING state. try { - final ContainerNodeSpec nodeSpec = nodeRepository.getContainer(hostname) + final ContainerNodeSpec nodeSpec = nodeRepository.getContainerNodeSpec(hostname) .orElseThrow(() -> new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname))); final Optional<Container> existingContainer = docker.getContainer(hostname); synchronizeLocalContainerState(nodeSpec, existingContainer); - } catch (Exception e) { - logger.log(LogLevel.ERROR, logPrefix + "Unhandled exception.", e); + } catch (RuntimeException e) { + logger.log(LogLevel.ERROR, logPrefix + "Unhandled exception, ignoring.", e); + } + catch (Throwable t) { + logger.log(LogLevel.ERROR, logPrefix + "Unhandled throwable, taking down system.", t); + System.exit(234); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java index 4ef7b1b0705..8ac7352799d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.docker; import com.google.common.base.Joiner; import com.google.common.io.CharStreams; +import com.google.inject.Inject; import com.spotify.docker.client.ContainerNotFoundException; import com.spotify.docker.client.DefaultDockerClient; import com.spotify.docker.client.DockerCertificateException; @@ -65,8 +66,8 @@ public class DockerImpl implements Docker { private static final int SECONDS_TO_WAIT_BEFORE_KILLING = 10; private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; - private static final String[] COMMAND_YINST_LS_VESPA = new String[]{"yinst", "ls", "vespa"}; - private static final Pattern VESPA_PACKAGE_VERSION_PATTERN = Pattern.compile("^vespa-(\\S+)", Pattern.MULTILINE); + static final String[] COMMAND_GET_VESPA_VERSION = new String[]{"vespa-nodectl", "vespa-version"}; + private static final Pattern VESPA_VERSION_PATTERN = Pattern.compile("^(\\S*)$", Pattern.MULTILINE); private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; private static final String LABEL_VALUE_MANAGEDBY = "node-admin"; @@ -79,9 +80,7 @@ public class DockerImpl implements Docker { } private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - private static final Path RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH = RELATIVE_APPLICATION_STORAGE_PATH.resolve("../container-storage-cleanup"); private static final Path APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_APPLICATION_STORAGE_PATH); - private static final Path CLEANUP_APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH); private static final Path APPLICATION_STORAGE_PATH_FOR_HOST = Paths.get("/").resolve(RELATIVE_APPLICATION_STORAGE_PATH); private static final List<String> DIRECTORIES_TO_MOUNT = Arrays.asList( @@ -101,7 +100,6 @@ public class DockerImpl implements Docker { getDefaults().underVespaHome("var/vespa"), getDefaults().underVespaHome("var/yca"), getDefaults().underVespaHome("var/ycore++"), - getDefaults().underVespaHome("var/ymon"), getDefaults().underVespaHome("var/zookeeper")); private final DockerClient docker; @@ -117,14 +115,16 @@ public class DockerImpl implements Docker { this.docker = dockerClient; } - public static DockerClient newDockerClientFromConfig(final DockerConfig config) { - return DefaultDockerClient.builder(). + @Inject + public DockerImpl(final DockerConfig config) { + this(DefaultDockerClient.builder(). uri(config.uri()). dockerCertificates(certificates(config)). readTimeoutMillis(TimeUnit.MINUTES.toMillis(30)). // Some operations may take minutes. - build(); + build()); } + private static DockerCertificates certificates(DockerConfig config) { try { return DockerCertificates.builder() @@ -205,7 +205,7 @@ public class DockerImpl implements Docker { log.log(LogLevel.INFO, "The application storage at " + from + " doesn't exist"); return; } - Path to = CLEANUP_APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN.resolve(containerName.asString() + "_" + filenameFormatter + Path to = applicationStoragePathForNodeAdmin("cleanup_" + containerName.asString() + "_" + filenameFormatter .format(Date.from(Instant.now()))); log.log(LogLevel.INFO, "Deleting application storage by moving it from " + from + " to " + to); Files.move(from, to); @@ -232,7 +232,7 @@ public class DockerImpl implements Docker { .networkMode("none") .binds(applicationStorageToMount(containerName.asString())) .build()) - .env("CONFIG_SERVER_ADDRESS=" + Joiner.on(',').join(Environment.getConfigServerHostsFromYinstSetting())). + .env("CONFIG_SERVER_ADDRESS=" + Joiner.on(',').join(Environment.getConfigServerHosts())). hostname(hostName.s()); if (minMainMemoryAvailableGb > 0.00001) { containerConfigBuilder.memory((long) (GIGA * minMainMemoryAvailableGb)); @@ -259,16 +259,23 @@ public class DockerImpl implements Docker { @Override public String getVespaVersion(final ContainerName containerName) { - ProcessResult result = executeInContainer(containerName, COMMAND_YINST_LS_VESPA); + ProcessResult result = executeInContainer(containerName, COMMAND_GET_VESPA_VERSION); if (!result.isSuccess()) { throw new RuntimeException("Container " + containerName.asString() + ": Command " - + Arrays.toString(COMMAND_YINST_LS_VESPA) + " failed: " + result); + + Arrays.toString(COMMAND_GET_VESPA_VERSION) + " failed: " + result); } - return parseVespaVersion(result.getOutput()) - .orElseThrow(() -> new RuntimeException( - "Container " + containerName.asString() + ": Failed to parse vespa version from " - + result.getOutput())); + .orElseThrow(() -> new RuntimeException( + "Container " + containerName.asString() + ": Failed to parse vespa version from " + + result.getOutput())); + } + + // Returns empty if vespa version cannot be parsed. + static Optional<String> parseVespaVersion(final String rawVespaVersion) { + if (rawVespaVersion == null) return Optional.empty(); + + final Matcher matcher = VESPA_VERSION_PATTERN.matcher(rawVespaVersion.trim()); + return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); } @Override @@ -298,12 +305,6 @@ public class DockerImpl implements Docker { } } - // Returns empty if vespa version cannot be parsed. - static Optional<String> parseVespaVersion(final String outputFromYinstLsVespa) { - final Matcher matcher = VESPA_PACKAGE_VERSION_PATTERN.matcher(outputFromYinstLsVespa); - return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); - } - private void setupContainerNetworking(ContainerName containerName, HostName hostName, int containerPid) throws UnknownHostException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java index 75d6f641feb..41312c28df7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import java.io.IOException; import java.util.Objects; public class ProcessResult { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java index 04d68269144..8829c3d4487 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java @@ -15,7 +15,7 @@ import java.util.Optional; public interface NodeRepository { List<ContainerNodeSpec> getContainersToRun() throws IOException; - Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException; + Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException; void updateNodeAttributes( HostName hostName, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java index 87b5ecd3a93..3d07309167c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java @@ -9,15 +9,15 @@ import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.GetNodesResponse; import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.NodeRepositoryApi; import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.UpdateNodeAttributesRequestBody; -import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.UpdateNodeAttributesResponse; import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory; import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory; import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory; +import javax.ws.rs.core.Response; import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -29,40 +29,18 @@ import java.util.logging.Logger; */ public class NodeRepositoryImpl implements NodeRepository { private static final Logger logger = Logger.getLogger(NodeRepositoryImpl.class.getName()); - private static final int HARDCODED_NODEREPOSITORY_PORT = 19071; private static final String NODEREPOSITORY_PATH_PREFIX_NODES_API = "/"; - private static final String ENV_HOSTNAME = "HOSTNAME"; private JaxRsStrategy<NodeRepositoryApi> nodeRepositoryClient; private final String baseHostName; - public NodeRepositoryImpl() { - baseHostName = Optional.ofNullable(System.getenv(ENV_HOSTNAME)) - .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); - nodeRepositoryClient = getApi(); - } - - // For testing - NodeRepositoryImpl(String baseHostName, String configserver, int configport) { - this.baseHostName = baseHostName; - final Set<HostName> configServerHosts = new HashSet<>(); - configServerHosts.add(new HostName(configserver)); - + public NodeRepositoryImpl(Set<HostName> configServerHosts, int configPort, String baseHostName) { final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( - configServerHosts, configport, jaxRsClientFactory); - nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); - } - - private static JaxRsStrategy<NodeRepositoryApi> getApi() { - final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting(); - if (configServerHosts.isEmpty()) { - throw new IllegalStateException("Environment setting for config servers missing or empty."); - } - final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); - final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( - configServerHosts, HARDCODED_NODEREPOSITORY_PORT, jaxRsClientFactory); - return jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); + configServerHosts, configPort, jaxRsClientFactory); + this.nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries( + NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); + this.baseHostName = baseHostName; } @Override @@ -92,11 +70,15 @@ public class NodeRepositoryImpl implements NodeRepository { } @Override - public Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException { - // TODO Use proper call to node repository - return getContainersToRun().stream() - .filter(cns -> Objects.equals(hostname, cns.hostname)) - .findFirst(); + public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException { + final GetNodesResponse response = nodeRepositoryClient.apply(nodeRepositoryApi -> nodeRepositoryApi.getNode(hostName.toString(), true)); + if (response.nodes.size() == 0) { + return Optional.empty(); + } + if (response.nodes.size() != 1) { + throw new RuntimeException("Did not get data for one node using hostname=" + hostName.toString() + "\n" + response.toString()); + } + return Optional.of(createContainerNodeSpec(response.nodes.get(0))); } private static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node) @@ -135,12 +117,20 @@ public class NodeRepositoryImpl implements NodeRepository { final String currentVespaVersion) throws IOException { // TODO: Filter out redundant (repeated) invocations with the same values. - // TODO: Error handling. - nodeRepositoryClient.apply(nodeRepositoryApi -> - nodeRepositoryApi.updateNodeAttributes( - hostName.s(), - new UpdateNodeAttributesRequestBody( - restartGeneration, dockerImage.asString(), currentVespaVersion))); + try { + nodeRepositoryClient.apply(nodeRepositoryApi -> + nodeRepositoryApi.updateNodeAttributes( + hostName.s(), + new UpdateNodeAttributesRequestBody( + restartGeneration, + dockerImage.asString(), + currentVespaVersion))); + } catch (javax.ws.rs.WebApplicationException e) { + final Response response = e.getResponse(); + UpdateNodeAttributesResponse updateResponse = response.readEntity(UpdateNodeAttributesResponse.class); + logger.log(LogLevel.ERROR, "Response code " + response.getStatus() + ": " + updateResponse.message); + throw new RuntimeException("Failed to update node attributes for " + hostName.s() + ":" + updateResponse.message); + } } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java index 36ab89a6718..7cecb37cd6d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java @@ -21,6 +21,17 @@ public interface NodeRepositoryApi { @QueryParam("parentHost") String hostname, @QueryParam("recursive") boolean recursive); + /** + * What is called "host" in NodeRepo is called "node" in node admin in this case. + * @param node the node to get data about. + * @param recursive set this to true, or you will not get the data you expect. + */ + @GET + @Path("/nodes/v2/node/") + GetNodesResponse getNode( + @QueryParam("hostname") String node, + @QueryParam("recursive") boolean recursive); + @PUT @Path("/nodes/v2/state/ready/{hostname}") // TODO: remove fake return String body; should be void and empty diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java index 9f2c7426875..b52397a1fff 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Automagically handles (de)serialization based on 1:1 message fields and identifier names. * Deserializes JSON strings on the form: @@ -12,6 +14,7 @@ package com.yahoo.vespa.hosted.node.admin.noderepository.bindings; * * @author bakksjo */ +@JsonIgnoreProperties(ignoreUnknown = true) public class UpdateNodeAttributesResponse { public String message; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java index 00365b5fc3d..392fed52ced 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.node.admin.orchestrator; import com.yahoo.vespa.applicationmodel.HostName; +import java.util.List; +import java.util.Optional; + /** * Abstraction for communicating with Orchestrator. * @@ -18,4 +21,9 @@ public interface Orchestrator { * Invokes orchestrator resume of a host. Returns whether resume was granted. */ boolean resume(HostName hostName); + + /** + * Invokes orchestrator suspend hosts. Returns failure reasons when failing. + */ + Optional<String> suspend(String parentHostName, List<String> hostNames); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java index 422962bdc58..907bc289544 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java @@ -7,6 +7,9 @@ import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory; import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory; import com.yahoo.vespa.orchestrator.restapi.HostApi; +import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; import com.yahoo.vespa.applicationmodel.HostName; @@ -14,7 +17,8 @@ import javax.ws.rs.ClientErrorException; import javax.ws.rs.NotFoundException; import javax.ws.rs.core.Response; import java.io.IOException; -import java.util.HashSet; +import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -22,6 +26,7 @@ import java.util.logging.Logger; /** * @author stiankri * @author bakksjo + * @author dybis */ public class OrchestratorImpl implements Orchestrator { private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName()); @@ -32,19 +37,21 @@ public class OrchestratorImpl implements Orchestrator { private static final String ORCHESTRATOR_PATH_PREFIX_HOST_API = ORCHESTRATOR_PATH_PREFIX + HostApi.PATH_PREFIX; - // We use this to allow client code to treat resume() calls as idempotent and cheap, - // but we actually filter out redundant resume calls to orchestrator. - private final Set<HostName> resumedHosts = new HashSet<>(); + private static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API + = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX; + private final JaxRsStrategy<HostApi> hostApiClient; + private final JaxRsStrategy<HostSuspensionApi> hostSuspensionClient; + - public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient) { + public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient, JaxRsStrategy<HostSuspensionApi> hostSuspensionClient) { this.hostApiClient = hostApiClient; + this.hostSuspensionClient = hostSuspensionClient; } @Override public boolean suspend(final HostName hostName) { - resumedHosts.remove(hostName); try { return hostApiClient.apply(api -> { final UpdateHostResponse response = api.suspend(hostName.s()); @@ -64,19 +71,33 @@ public class OrchestratorImpl implements Orchestrator { } @Override - public boolean resume(final HostName hostName) { - if (resumedHosts.contains(hostName)) { - return true; + public Optional<String> suspend(String parentHostName, List<String> hostNames) { + try { + return hostSuspensionClient.apply(hostSuspensionClient -> { + BatchHostSuspendRequest request = new BatchHostSuspendRequest(parentHostName, hostNames); + final BatchOperationResult result = hostSuspensionClient.suspendAll(request); + return result.getFailureReason(); + }); + } catch (ClientErrorException e) { + if (e instanceof NotFoundException || e.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) { + // Orchestrator doesn't care about this node, so don't let that stop us. + return Optional.empty(); + } + logger.log(Level.INFO, "Orchestrator rejected suspend request for host " + parentHostName, e); + return Optional.of(e.getLocalizedMessage()); + } catch (IOException e) { + logger.log(Level.WARNING, "Unable to communicate with orchestrator", e); + return Optional.of("Unable to communicate with orchestrator" + e.getMessage()); } + } + @Override + public boolean resume(final HostName hostName) { try { final boolean resumeSucceeded = hostApiClient.apply(api -> { final UpdateHostResponse response = api.resume(hostName.s()); return response.reason() == null; }); - if (resumeSucceeded) { - resumedHosts.add(hostName); - } return resumeSucceeded; } catch (ClientErrorException e) { logger.log(Level.INFO, "Orchestrator rejected resume request for host " + hostName, e); @@ -87,15 +108,16 @@ public class OrchestratorImpl implements Orchestrator { } } - public static JaxRsStrategy<HostApi> makeOrchestratorHostApiClient() { - final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting(); + public static OrchestratorImpl createOrchestratorFromSettings() { + final Set<HostName> configServerHosts = Environment.getConfigServerHosts(); if (configServerHosts.isEmpty()) { - throw new IllegalStateException("Emnvironment setting for config servers missing or empty."); + throw new IllegalStateException("Environment setting for config servers missing or empty."); } final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( configServerHosts, HARDCODED_ORCHESTRATOR_PORT, jaxRsClientFactory); - return jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API); + JaxRsStrategy<HostApi> hostApi = jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API); + JaxRsStrategy<HostSuspensionApi> suspendApi = jaxRsStrategyFactory.apiWithRetries(HostSuspensionApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API); + return new OrchestratorImpl(hostApi, suspendApi); } - } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java new file mode 100644 index 00000000000..343ac24b20f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java @@ -0,0 +1,13 @@ +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; + +/** + * Class for setting up instances of classes; enables testing. + * + * @author dybis + */ +public interface ComponentsProvider { + NodeAdminStateUpdater getNodeAdminStateUpdater(); + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java new file mode 100644 index 00000000000..fdc72d973cd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.Docker; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.util.Set; +import java.util.function.Function; + +/** + * Set up node admin for production. + * + * @author dybis + */ +public class ComponentsProviderImpl implements ComponentsProvider { + + private final Docker docker; + private static final long INITIAL_SCHEDULER_DELAY_MILLIS = 0; + private static final long INTERVAL_SCHEDULER_IN_MILLIS = 60000; + + private static final int HARDCODED_NODEREPOSITORY_PORT = 19071; + private static final String ENV_HOSTNAME = "HOSTNAME"; + public ComponentsProviderImpl(final Docker docker) { + this.docker = docker; + } + + @Override + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + String baseHostName = java.util.Optional.ofNullable(System.getenv(ENV_HOSTNAME)) + .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); + + Set<HostName> configServerHosts = Environment.getConfigServerHosts(); + if (configServerHosts.isEmpty()) { + throw new IllegalStateException("Environment setting for config servers missing or empty."); + } + + NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, HARDCODED_NODEREPOSITORY_PORT, baseHostName); + + Orchestrator orchestrator = OrchestratorImpl.createOrchestratorFromSettings(); + final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> + new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator); + final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory); + return new NodeAdminStateUpdater( + nodeRepository, nodeAdmin, INITIAL_SCHEDULER_DELAY_MILLIS, INTERVAL_SCHEDULER_IN_MILLIS, orchestrator, baseHostName); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java new file mode 100644 index 00000000000..468577e8f4f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java deleted file mode 100644 index 9c2af68d56a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.restapi; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; - -/** - * @author stiankri - */ -@Path("") -public interface NodeAdminRestAPI { - @GET - @Path("/update") - public String update(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java new file mode 100644 index 00000000000..ace3fbfc764 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -0,0 +1,102 @@ +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; + +/** + * Rest API for suspending and resuming the docker host. + * There are two non-blocking idempotent calls: /resume and /suspend. + * + * There is one debug call: /info + * + * @author dybis + */ +public class RestApiHandler extends LoggingRequestHandler{ + + private final NodeAdminStateUpdater refresher; + private final static ObjectMapper objectMapper = new ObjectMapper(); + + public RestApiHandler(Executor executor, AccessLog accessLog, ComponentsProvider componentsProvider) { + super(executor, accessLog); + this.refresher = componentsProvider.getNodeAdminStateUpdater(); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getMethod() == GET) { + return handleGet(request); + } + if (request.getMethod() == PUT) { + return handlePut(request); + } + return new SimpleResponse(400, "Only PUT and GET are implemented."); + + } + + private HttpResponse handleGet(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.endsWith("/info")) { + return new SimpleResponse(200, refresher.getDebugPage()); + } + return new SimpleResponse(400, "unknown path" + path); + } + + private HttpResponse handlePut(HttpRequest request) { + String path = request.getUri().getPath(); + // Check paths to disallow illegal state changes + if (path.endsWith("/resume")) { + final Optional<String> errorMessage = refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); + if (errorMessage.isPresent()) { + return new SimpleResponse(409, errorMessage.get()); + } + return new SimpleResponse(200, "ok"); + } + if (path.endsWith("/suspend")) { + Optional<String> errorMessage = refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); + if (errorMessage.isPresent()) { + return new SimpleResponse(409, errorMessage.get()); + } + return new SimpleResponse(200, "ok"); + } + return new SimpleResponse(400, "unknown path" + path); + } + + private static class SimpleResponse extends HttpResponse { + + private final String jsonMessage; + + public SimpleResponse(int code, String message) { + super(code); + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.put("jsonMessage", message); + this.jsonMessage = objectNode.toString(); + } + + @Override + public String getContentType() { + return MediaType.APPLICATION_JSON; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(jsonMessage.getBytes(StandardCharsets.UTF_8.name())); + } + } + +} diff --git a/vespalib/src/tests/placement-delete/fail.cpp b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java index b6df97df6d4..5e5bd002950 100644 --- a/vespalib/src/tests/placement-delete/fail.cpp +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java @@ -1,2 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -This should not compile with any version of gcc... +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java deleted file mode 100644 index 65da2004854..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.testapi; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -/** - * Resource for use in integration test, will be deleted soon. - * @author tonytv - */ -@Path("ping") -public class PingResource { - @GET - @Produces(MediaType.TEXT_PLAIN) - public String ping() { - return "pong"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index d6703326796..b5929653458 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -22,13 +22,13 @@ public class Environment { public enum NetworkType { normal, local, vm } - public static Set<HostName> getConfigServerHostsFromYinstSetting() { - final String yinstSetting = System.getenv(ENV_CONFIGSERVERS); - if (yinstSetting == null) { + public static Set<HostName> getConfigServerHosts() { + final String configServerHosts = System.getenv(ENV_CONFIGSERVERS); + if (configServerHosts == null) { return Collections.emptySet(); } - final List<String> hostNameStrings = Arrays.asList(yinstSetting.split("[,\\s]+")); + final List<String> hostNameStrings = Arrays.asList(configServerHosts.split("[,\\s]+")); return hostNameStrings.stream() .map(HostName::new) .collect(Collectors.toSet()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java index a4843a0753e..737dd25092a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java @@ -34,7 +34,7 @@ import static org.mockito.Mockito.when; /** * @author bakksjo */ -public class NodeAdminTest { +public class NodeAdminImplTest { private static final Optional<Double> MIN_CPU_CORES = Optional.of(1.0); private static final Optional<Double> MIN_MAIN_MEMORY_AVAILABLE_GB = Optional.of(1.0); private static final Optional<Double> MIN_DISK_AVAILABLE_GB = Optional.of(1.0); @@ -47,7 +47,7 @@ public class NodeAdminTest { final Docker docker = mock(Docker.class); final Function<HostName, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory); + final NodeAdminImpl nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory); final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class); final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class); @@ -76,30 +76,30 @@ public class NodeAdminTest { nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory).apply(hostName); inOrder.verify(nodeAgent1).start(); - inOrder.verify(nodeAgent1).update(); + inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); inOrder.verify(nodeAgent1, never()).stop(); nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1).update(); + inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); inOrder.verify(nodeAgent1, never()).stop(); nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), asList(existingContainer)); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); - inOrder.verify(nodeAgent1, never()).update(); + inOrder.verify(nodeAgent1, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); verify(nodeAgent1).stop(); nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory).apply(hostName); inOrder.verify(nodeAgent2).start(); - inOrder.verify(nodeAgent2).update(); + inOrder.verify(nodeAgent2).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); inOrder.verify(nodeAgent2, never()).stop(); nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), Collections.emptyList()); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent2, never()).update(); + inOrder.verify(nodeAgent2, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); inOrder.verify(nodeAgent2).stop(); verifyNoMoreInteractions(nodeAgent1); @@ -116,7 +116,7 @@ public class NodeAdminTest { final Set<DockerImage> currentlyUnusedImages = Collections.emptySet(); final List<ContainerNodeSpec> pendingContainers = Collections.emptyList(); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); assertThat(deletableImages, is(Collections.emptySet())); } @@ -127,7 +127,7 @@ public class NodeAdminTest { .collect(Collectors.toSet()); final List<ContainerNodeSpec> pendingContainers = Collections.emptyList(); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3) .collect(Collectors.toSet()); @@ -139,10 +139,10 @@ public class NodeAdminTest { final Set<DockerImage> currentlyUnusedImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3) .collect(Collectors.toSet()); final List<ContainerNodeSpec> pendingContainers = Stream.of(IMAGE_2, IMAGE_4) - .map(NodeAdminTest::newNodeSpec) + .map(NodeAdminImplTest::newNodeSpec) .collect(Collectors.toList()); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_3) .collect(Collectors.toSet()); @@ -168,7 +168,7 @@ public class NodeAdminTest { newPair(null, 21))); assertThat( - NodeAdmin.fullOuterJoin( + NodeAdminImpl.fullOuterJoin( strings.stream(), string -> string, integers.stream(), String::valueOf) .collect(Collectors.toSet()), diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java new file mode 100644 index 00000000000..2b8517ab33b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java @@ -0,0 +1,84 @@ +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.integrationTests.OrchestratorMock; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; + +import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.junit.MatcherAssert.assertThat; +import static org.mockito.Matchers.anyList; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Basic test of ActiveContainersRefresherTest + * @author dybis + */ +public class NodeAdminStateUpdaterTest { + + @Test + @SuppressWarnings("unchecked") + public void testExceptionIsCaughtAndDataIsPassedAndFreeze() throws Exception { + NodeRepository nodeRepository = mock(NodeRepository.class); + NodeAdmin nodeAdmin = mock(NodeAdmin.class); + final List<ContainerNodeSpec> accumulatedArgumentList = Collections.synchronizedList(new ArrayList<>()); + final CountDownLatch latch = new CountDownLatch(5); + doAnswer((Answer<Object>) invocation -> { + List<ContainerNodeSpec> containersToRunInArgument = (List<ContainerNodeSpec>) invocation.getArguments()[0]; + containersToRunInArgument.forEach(element -> accumulatedArgumentList.add(element)); + latch.countDown(); + if (accumulatedArgumentList.size() == 2) { + throw new RuleBaseException("This exception is expected, and should show up in the log."); + } + return null; + }).when(nodeAdmin).refreshContainersToRun(anyList()); + + final List<ContainerNodeSpec> containersToRun = new ArrayList<>(); + containersToRun.add(createSample()); + + when(nodeRepository.getContainersToRun()).thenReturn(containersToRun); + OrchestratorMock orchestratorMock = new OrchestratorMock(); + NodeAdminStateUpdater refresher = new NodeAdminStateUpdater( + nodeRepository, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + latch.await(); + int numberOfElements = accumulatedArgumentList.size(); + assertThat(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), + is(Optional.of("Not all node agents are frozen."))); + assertTrue(numberOfElements > 4); + assertThat(accumulatedArgumentList.get(0), is(createSample())); + Thread.sleep(2); + assertThat(accumulatedArgumentList.size(), is(numberOfElements)); + assertThat(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), + is(Optional.empty())); + while (accumulatedArgumentList.size() == numberOfElements) { + Thread.sleep(1); + } + refresher.deconstruct(); + } + + private ContainerNodeSpec createSample() { + return new ContainerNodeSpec( + new HostName("hostname"), + Optional.empty(), + new ContainerName("containername"), + NodeState.ACTIVE, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java index 9351e80cdab..70f24304edd 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java @@ -87,15 +87,25 @@ public class DockerImplTest { @Test public void vespaVersionIsParsed() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53"), is(Optional.of("5.119.53"))); + assertThat(DockerImpl.parseVespaVersion("5.119.53"), is(Optional.of("5.119.53"))); } @Test - public void vespaVersionIsParsedWithTrailingNewline() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53\n"), is(Optional.of("5.119.53"))); + public void vespaVersionIsParsedWithSpacesAndNewlines() { + assertThat(DockerImpl.parseVespaVersion("5.119.53\n"), is(Optional.of("5.119.53"))); + assertThat(DockerImpl.parseVespaVersion(" 5.119.53 \n"), is(Optional.of("5.119.53"))); + assertThat(DockerImpl.parseVespaVersion("\n 5.119.53 \n"), is(Optional.of("5.119.53"))); } - @Test(expected = NullPointerException.class) + @Test + public void vespaVersionIsParsedWithIrregularVersionScheme() { + assertThat(DockerImpl.parseVespaVersion("7.2"), is(Optional.of("7.2"))); + assertThat(DockerImpl.parseVespaVersion("8.0-beta"), is(Optional.of("8.0-beta"))); + assertThat(DockerImpl.parseVespaVersion("foo"), is(Optional.of("foo"))); + assertThat(DockerImpl.parseVespaVersion("119"), is(Optional.of("119"))); + } + + @Test public void vespaVersionIsNotParsedFromNull() { assertThat(DockerImpl.parseVespaVersion(null), is(Optional.empty())); } @@ -107,11 +117,7 @@ public class DockerImplTest { @Test public void vespaVersionIsNotParsedFromUnexpectedContent() { - assertThat(DockerImpl.parseVespaVersion("honda-5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa 5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa- 5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa-"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("No such command 'yinst'"), is(Optional.empty())); + assertThat(DockerImpl.parseVespaVersion("No such command 'vespanodectl'"), is(Optional.empty())); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java new file mode 100644 index 00000000000..cf147ce7936 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java @@ -0,0 +1,20 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; + +/** + * For setting up test with mocks. + * + * @author dybis + */ +public class ComponentsProviderWithMocks implements ComponentsProvider { + private NodeRepoMock nodeRepositoryMock = new NodeRepoMock(); + private NodeAdminMock nodeAdminMock = new NodeAdminMock(); + private OrchestratorMock orchestratorMock = new OrchestratorMock(); + + @Override + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdminMock, 1, 5, orchestratorMock, "hostname"); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java new file mode 100644 index 00000000000..0f042162de3 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java @@ -0,0 +1,64 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Mock with some simple logic + * + * @author dybis + */ +public class NodeAdminMock implements NodeAdmin { + + StringBuilder info = new StringBuilder(); + + Set<HostName> hostnames = new HashSet<>(); + + boolean freezeSetState = false; + public AtomicBoolean frozen = new AtomicBoolean(false); + + // We make it threadsafe as the test have its own peeking thread. + private Object monitor = new Object(); + + @Override + public void refreshContainersToRun(List<ContainerNodeSpec> containersToRun) { + synchronized (monitor) { + hostnames.clear(); + containersToRun.forEach(container -> hostnames.add(container.hostname)); + } + } + + @Override + public boolean freezeAndCheckIfAllFrozen() { + info.append(" Freeze called while in state " + frozen.get()); + freezeSetState = true; + return frozen.get(); + } + + @Override + public void unfreeze() { + info.append(" Unfreeze called while in state " + frozen.get()); + freezeSetState = false; + } + + @Override + public Set<HostName> getListOfHosts() { + synchronized (monitor) { + return hostnames; + } + } + + /* + * We use this to get some information easily out of the mock in the integration test here. + */ + @Override + public String debugInfo() { + return info.toString(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java new file mode 100644 index 00000000000..1d4bb297e49 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Mock with some simple logic + * @author dybis + */ +public class NodeRepoMock implements NodeRepository { + + public List<ContainerNodeSpec> containerNodeSpecs = new ArrayList<>(); + + @Override + public List<ContainerNodeSpec> getContainersToRun() throws IOException { + return containerNodeSpecs; + } + + @Override + public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException { + return null; + } + + @Override + public void updateNodeAttributes(HostName hostName, long restartGeneration, DockerImage dockerImage, String containerVespaVersion) throws IOException { + + } + + @Override + public void markAsReady(HostName hostName) throws IOException { + + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java new file mode 100644 index 00000000000..eccf12daaeb --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; + +import java.util.List; +import java.util.Optional; + +/** + * Mock with some simple logic + * @author dybis + */ +public class OrchestratorMock implements Orchestrator { + + public Optional<String> suspendReturnValue = Optional.empty(); + + @Override + public boolean suspend(HostName hostName) { + return false; + } + + @Override + public boolean resume(HostName hostName) { + return false; + } + + @Override + public Optional<String> suspend(String parentHostName, List<String> hostNames) { + return suspendReturnValue; + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java new file mode 100644 index 00000000000..e9d367a3f59 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java @@ -0,0 +1,76 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import org.junit.Test; + +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * Scenario test for NodeAdminStateUpdater. + * @author dybis + */ +public class ResumeTest { + @Test + public void test() throws InterruptedException { + NodeRepoMock nodeRepositoryMock = new NodeRepoMock(); + NodeAdminMock nodeAdminMock = new NodeAdminMock(); + OrchestratorMock orchestratorMock = new OrchestratorMock(); + + nodeRepositoryMock.containerNodeSpecs.add(new ContainerNodeSpec( + new HostName("hostname"), + Optional.of(new DockerImage("dockerimage")), + new ContainerName("containe"), + NodeState.ACTIVE, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty())); + + NodeAdminStateUpdater updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdminMock, 1, 1, orchestratorMock, "basehostname"); + // Wait for node admin to be notified with node repo state + while (nodeAdminMock.getListOfHosts().size() == 0) { + Thread.sleep(1); + } + + // Make node admin and orchestrator block suspend + orchestratorMock.suspendReturnValue = Optional.of("orch reject suspend"); + nodeAdminMock.frozen.set(false); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("Not all node agents are frozen."))); + + // Now, change data in node repo, should not propagate. + nodeRepositoryMock.containerNodeSpecs.clear(); + + // Set node admin not blocking for suspend + nodeAdminMock.frozen.set(true); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("orch reject suspend"))); + + // Make orchestrator allow suspend + orchestratorMock.suspendReturnValue = Optional.empty(); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.empty())); + + // Now suspended, new node repo state should have not propagated to node admin + Thread.sleep(2); + assertThat(nodeAdminMock.getListOfHosts().size(), is(1)); + + // Now resume + nodeAdminMock.frozen.set(false); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), is(Optional.empty())); + + // Now node repo state should propagate to node admin again + while (nodeAdminMock.getListOfHosts().size() != 0) { + Thread.sleep(1); + } + + updater.deconstruct(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java new file mode 100644 index 00000000000..21862f4598f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -0,0 +1,129 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.google.common.collect.Sets; +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author dybis + */ +public class RunInContainerTest { + + private JDisc container; + private int port; + + private int findRandomOpenPort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + @Before + public void startContainer() throws Exception { + port = findRandomOpenPort(); + System.out.println("PORT IS " + port); + container = JDisc.fromServicesXml(createServiceXml(port), Networking.enable); + } + + private boolean doPutCall(String command) throws IOException { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpPut getRequest = new HttpPut("/rest/" + command); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + return httpResponse.getStatusLine().getStatusCode() == 200; + } + + private String doGetInfoCall() throws IOException { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpGet getRequest = new HttpGet("/rest/info"); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + HttpEntity entity = httpResponse.getEntity(); + StringWriter writer = new StringWriter(); + IOUtils.copy(entity.getContent(), writer, StandardCharsets.UTF_8); + return writer.toString(); + } + + private void waitForJdiscContainerToServe() throws InterruptedException { + Instant start = Instant.now(); + while (Instant.now().minusSeconds(120).isBefore(start)) { + try { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpGet getRequest = new HttpGet("/rest/info"); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + HttpEntity entity = httpResponse.getEntity(); + if (httpResponse.getStatusLine().getStatusCode() != 200) { + continue; + } + System.out.println("Container started."); + return; + } catch (Exception e) { + Thread.sleep(100); + } + } + throw new RuntimeException("Could not get answer from container."); + } + + @After + public void stopContainer() { + if (container != null) { + container.close(); + } + } + + @Test + public void testGetContainersToRunAPi() throws IOException, InterruptedException { + waitForJdiscContainerToServe(); + assertThat(doPutCall("resume"), is(true)); + assertThat(doPutCall("suspend"), is(false)); + assertThat(doGetInfoCall(), is("{\"jsonMessage\":\"isRunningUpdates is false. NodeAdmin: " + + "Unfreeze called while in state false " + + "Freeze called while in state false\"}")); + } + + + private String createServiceXml(int port) { + return "<services version=\"1.0\">\n" + + " <jdisc version=\"1.0\" jetty=\"true\">\n" + + " <handler id=\"com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler\" bundle=\"node-admin\">\n" + + " <binding>http://*/rest/*</binding>\n" + + " </handler>\n" + + " <component id=\"node-admin\" class=\"com.yahoo.vespa.hosted.node.admin.integrationTests.ComponentsProviderWithMocks\" bundle=\"node-admin\"/>\n" + + " <http>" + + " <server id=\'myServer\' port=\'" + port + "\' />" + + " </http>" + + " </jdisc>\n" + + "</services>\n"; + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 633405cc5f7..a4b250c6cea 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.hosted.node.admin.noderepository; +import com.google.common.collect.Sets; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.vespa.applicationmodel.HostName; @@ -8,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -17,14 +20,22 @@ import java.net.ServerSocket; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; - +/** + * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository + * which already contains some data. + * + * @author dybdahl + */ public class NodeRepositoryImplTest { private JDisc container; private int port; + private final Set<HostName> configServerHosts = Sets.newHashSet(new HostName("127.0.0.1")); + private int findRandomOpenPort() throws IOException { try (ServerSocket socket = new ServerSocket(0)) { @@ -42,12 +53,13 @@ public class NodeRepositoryImplTest { @Before public void startContainer() throws Exception { port = findRandomOpenPort(); + System.err.println("PORT IS " + port); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); } private void waitForJdiscContainerToServe() throws InterruptedException { Instant start = Instant.now(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("foobar", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(Sets.newHashSet(new HostName("127.0.0.1")), port, "foobar"); while (Instant.now().minusSeconds(120).isBefore(start)) { try { nodeRepositoryApi.getContainersToRun(); @@ -66,11 +78,10 @@ public class NodeRepositoryImplTest { } } - @Test public void testGetContainersToRunAPi() throws IOException, InterruptedException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(); assertThat(containersToRun.size(), is(1)); final ContainerNodeSpec nodeSpec = containersToRun.get(0); @@ -88,11 +99,28 @@ public class NodeRepositoryImplTest { @Test public void testGetContainers() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); HostName hostname = new HostName("host4.yahoo.com"); - Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainer(hostname); + Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertThat(nodeSpec.isPresent(), is(true)); assertThat(nodeSpec.get().hostname, is(hostname)); assertThat(nodeSpec.get().containerName, is(new ContainerName("host4"))); } + + @Test + public void testUpdateNodeAttributes() throws InterruptedException, IOException { + waitForJdiscContainerToServe(); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); + HostName hostname = new HostName("host4.yahoo.com"); + nodeRepositoryApi.updateNodeAttributes(hostname, 1L, new DockerImage("image-1"), "6.2.3"); + } + + @Test(expected = RuntimeException.class) + public void testUpdateNodeAttributesWithBadValue() throws InterruptedException, IOException { + waitForJdiscContainerToServe(); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); + HostName hostname = new HostName("host4.yahoo.com"); + nodeRepositoryApi.updateNodeAttributes(hostname, 1L, new DockerImage("image-1"), "6.2.3\n"); + } + } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java index f5c5601d661..1b7e071889e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java @@ -5,10 +5,22 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy; import com.yahoo.vespa.orchestrator.restapi.HostApi; +import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; +import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -import static org.mockito.Matchers.anyString; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -16,16 +28,29 @@ import static org.mockito.Mockito.when; /** * @author bakksjo + * @author dybis */ public class OrchestratorImplTest { - @Test - public void redundantResumesAreFilteredOut() throws Exception { - final HostApi hostApi = mock(HostApi.class); + private HostApi hostApi; + private OrchestratorImpl orchestrator; + private HostSuspensionApi hostSuspensionApi; + final String hostNameString = "host"; + final HostName hostName = new HostName(hostNameString); + final List<String> hosts = new ArrayList<>(); + + @Before + public void before() { + hostApi = mock(HostApi.class); final JaxRsStrategy<HostApi> hostApiClient = new LocalPassThroughJaxRsStrategy<>(hostApi); - final OrchestratorImpl orchestrator = new OrchestratorImpl(hostApiClient); - final String hostNameString = "host"; - final HostName hostName = new HostName(hostNameString); + hostSuspensionApi = mock(HostSuspensionApi.class); + final JaxRsStrategy<HostSuspensionApi> hostSuspendClient = new LocalPassThroughJaxRsStrategy<>(hostSuspensionApi); + + orchestrator = new OrchestratorImpl(hostApiClient, hostSuspendClient); + } + + @Test + public void testSingleOperations() throws Exception { // Make resume and suspend always succeed. when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null)); when(hostApi.suspend(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null)); @@ -33,18 +58,23 @@ public class OrchestratorImplTest { orchestrator.resume(hostName); verify(hostApi, times(1)).resume(hostNameString); - // A subsequent resume does not cause a network trip. - orchestrator.resume(hostName); - verify(hostApi, times(1)).resume(anyString()); - orchestrator.suspend(hostName); verify(hostApi, times(1)).suspend(hostNameString); - orchestrator.resume(hostName); - verify(hostApi, times(2)).resume(hostNameString); + hosts.add(hostNameString); + } - // A subsequent resume does not cause a network trip. - orchestrator.resume(hostName); - verify(hostApi, times(2)).resume(anyString()); + @Test + public void testListSuspendOk() throws Exception { + hosts.add(hostNameString); + when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult(null)); + assertThat(orchestrator.suspend("parent", hosts), is(Optional.empty())); + } + + @Test + public void testListSuspendFailed() throws Exception { + hosts.add(hostNameString); + when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult("no no")); + assertThat(orchestrator.suspend("parent", hosts), is(Optional.of("no no"))); } } diff --git a/node-repository/pom.xml b/node-repository/pom.xml index da48ff9eb4b..bb3661f8042 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-repository</artifactId> <version>6-SNAPSHOT</version> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 2ab98f0c582..0456f6071c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -30,6 +30,7 @@ public final class Node { private final Configuration configuration; private final Status status; private final State state; + private final Type type; /** Record of the last event of each type happening to this node */ private final History history; @@ -38,19 +39,20 @@ public final class Node { private Optional<Allocation> allocation; /** Creates a node in the initial state (provisioned) */ - public static Node create(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) { + public static Node create(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration, Type type) { return new Node(openStackId, hostname, parentHostname, configuration, Status.initial(), State.provisioned, - Optional.empty(), History.empty()); + Optional.empty(), History.empty(), type); } /** Do not use. Construct nodes by calling {@link NodeRepository#createNode} */ public Node(String openStackId, String hostname, Optional<String> parentHostname, - Configuration configuration, Status status, State state, Allocation allocation, History history) { - this(openStackId, hostname, parentHostname, configuration, status, state, Optional.of(allocation), history); + Configuration configuration, Status status, State state, Allocation allocation, History history, Type type) { + this(openStackId, hostname, parentHostname, configuration, status, state, Optional.of(allocation), history, type); } public Node(String openStackId, String hostname, Optional<String> parentHostname, - Configuration configuration, Status status, State state, Optional<Allocation> allocation, History history) { + Configuration configuration, Status status, State state, Optional<Allocation> allocation, + History history, Type type) { Objects.requireNonNull(openStackId, "A node must have an openstack id"); Objects.requireNonNull(hostname, "A node must have a hostname"); Objects.requireNonNull(parentHostname, "A null parentHostname is not permitted."); @@ -59,6 +61,7 @@ public final class Node { Objects.requireNonNull(state, "A null node state is not permitted"); Objects.requireNonNull(allocation, "A null node allocation is not permitted"); Objects.requireNonNull(history, "A null node history is not permitted"); + Objects.requireNonNull(type, "A null node type is not permitted"); this.id = hostname; this.hostname = hostname; @@ -69,6 +72,7 @@ public final class Node { this.state = state; this.allocation = allocation; this.history = history; + this.type = type; } /** @@ -96,6 +100,9 @@ public final class Node { /** Returns the current state of this node (in the node state machine) */ public State state() { return state; } + /** Returns the type of this node */ + public Type type() { return type; } + /** Returns the current allocation of this, if any */ public Optional<Allocation> allocation() { return allocation; } @@ -135,24 +142,29 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node setStatus(Status status) { - return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history, type); + } + + /** Returns a node with the type assigned to the given value */ + public Node setType(Type type) { + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history, type); } /** Returns a node with the hardware configuration assigned to the given value */ public Node setConfiguration(Configuration configuration) { - return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history, type); } /** Returns a copy of this with the current generation set to generation */ public Node setReboot(Generation generation) { return new Node(openStackId, hostname, parentHostname, configuration, status.setReboot(generation), state, - allocation, history); + allocation, history, type); } /** Returns a copy of this with the flavor set to flavor */ public Node setFlavor(Flavor flavor) { return new Node(openStackId, hostname, parentHostname, new Configuration(flavor), status, state, - allocation, history); + allocation, history, type); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -176,17 +188,17 @@ public final class Node { * Do not use this to allocate a node. */ public Node setAllocation(Allocation allocation) { - return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history, type); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node setParentHostname(String parentHostname) { - return new Node(openStackId, hostname, Optional.of(parentHostname), configuration, status, state, allocation, history); + return new Node(openStackId, hostname, Optional.of(parentHostname), configuration, status, state, allocation, history, type); } /** Returns a copy of this node with the given history. */ private Node setHistory(History history) { - return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history); + return new Node(openStackId, hostname, parentHostname, configuration, status, state, allocation, history, type); } @Override @@ -230,13 +242,24 @@ public final class Node { dirty, /** This node has failed and must be repaired or removed. The node retains any allocation data for diagnosis. */ - failed; + failed, + + /** + * This node should not currently be used. + * This state follows the same rules as failed except that it will never be automatically moved out of + * this state. + */ + parked; /** Returns whether this is a state where the node is assigned to an application */ public boolean isAllocated() { - return this == reserved || this == active || this == inactive || this == failed; + return this == reserved || this == active || this == inactive || this == failed || this == parked; } + } + public enum Type { + tenant, + host; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1980b1f5318..11892c69c1c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; // 1) (new) - > provisioned -> ready -> reserved -> active -> inactive -> dirty -> ready // 2) inactive -> reserved // 3) reserved -> dirty -// 3) * -> failed -> dirty | active | (removed) +// 3) * -> failed | parked -> dirty | active | (removed) // Nodes have an application assigned when in states reserved, active and inactive. // Nodes might have an application assigned in dirty. public class NodeRepository extends AbstractComponent { @@ -94,7 +94,9 @@ public class NodeRepository extends AbstractComponent { return zkClient.getNode(state, hostname); } - public List<Node> getNodes(Node.State ... inState) { return zkClient.getNodes(inState); } + public List<Node> getNodes(Node.Type type, Node.State ... inState) { + return zkClient.getNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList()); + } public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return zkClient.getNodes(id, inState); } public List<Node> getInactive() { return zkClient.getNodes(Node.State.inactive); } public List<Node> getFailed() { return zkClient.getNodes(Node.State.failed); } @@ -108,8 +110,9 @@ public class NodeRepository extends AbstractComponent { // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo */ - public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Configuration configuration) { - return Node.create(openStackId, hostname, parentHostname, configuration); + public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, + Configuration configuration, Node.Type type) { + return Node.create(openStackId, hostname, parentHostname, configuration, type); } /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ @@ -177,11 +180,16 @@ public class NodeRepository extends AbstractComponent { return performOn(NodeListFilter.from(nodes), node -> zkClient.writeTo(Node.State.dirty, node)); } - /** Deallocate a node which is in the failed state. Use this to recycle failed nodes which have been repaired. */ + /** + * Deallocate a node which is in the failed or parked state. + * Use this to recycle failed nodes which have been repaired or put on hold. + */ public Node deallocate(String hostname) { Optional<Node> nodeToDeallocate = getNode(Node.State.failed, hostname); if ( ! nodeToDeallocate.isPresent()) - throw new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found in the failed state"); + nodeToDeallocate = getNode(Node.State.parked, hostname); + if ( ! nodeToDeallocate.isPresent()) + throw new IllegalArgumentException("Could not deallocate " + hostname + ": No such node in the failed or parked state"); return deallocate(Collections.singletonList(nodeToDeallocate.get())).get(0); } @@ -196,12 +204,22 @@ public class NodeRepository extends AbstractComponent { } /** - * Moves a previously failed node back to the active state. + * Parks this node and returns it in its new state. + * + * @return the node in its new state + * @throws IllegalArgumentException if the node is not found + */ + public Node park(String hostname) { + return move(hostname, Node.State.parked); + } + + /** + * Moves a previously failed or parked node back to the active state. * * @return the node in its new state * @throws IllegalArgumentException if the node is not found */ - public Node unfail(String hostname) { + public Node reactivate(String hostname) { return move(hostname, Node.State.active); } @@ -215,15 +233,18 @@ public class NodeRepository extends AbstractComponent { } /** - * Removes a node. A node must be in the failed state before it can be removed. + * Removes a node. A node must be in the failed or parked state before it can be removed. * * @return true if the node was removed, false if it was not found */ public boolean remove(String hostname) { Optional<Node> nodeToRemove = getNode(Node.State.failed, hostname); - if ( ! nodeToRemove.isPresent()) return false; + if ( ! nodeToRemove.isPresent()) + nodeToRemove = getNode(Node.State.parked, hostname); + if ( ! nodeToRemove.isPresent()) + return false; try (Mutex lock = lock(nodeToRemove.get())) { - return zkClient.removeNode(Node.State.failed, hostname); + return zkClient.removeNode(nodeToRemove.get().state(), hostname); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java index 1ec20f6df0c..37e0fb2da17 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/assimilate/PopulateClient.java @@ -85,14 +85,15 @@ public class PopulateClient { private Node buildNode(String hostname, String clusterType, String clusterId, int nodeIndex) { return new Node( - hostname, // Id - hostname, // Hostname - Optional.empty(), // parent hostname - new Configuration(getFlavor(clusterType, clusterId).get()), // Flavor + hostname /* id */, + hostname /* Hostname */, + Optional.empty() /* parent hostname */, + new Configuration(getFlavor(clusterType, clusterId).get()), Status.initial(), - Node.State.active, // State = active/allocated - Optional.empty(), // Allocation - History.empty()) // History + Node.State.active, + Optional.empty() /* Allocation */, + History.empty(), + Node.Type.tenant) // History .allocate( ApplicationId.from( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index fd5e229fa1b..ce7ae429c40 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -18,7 +18,7 @@ import java.util.stream.Collectors; * The application maintainer regularly redeploys all applications. * This is necessary because applications may gain and lose active nodes due to nodes being moved to and from the * failed state. This is corrected by redeploying the applications periodically. - * It can not (at this point) be done reliably synchronously as part of the fail/unfail call due to the need for this + * It can not (at this point) be done reliably synchronously as part of the fail/reactivate call due to the need for this * to happen at a node having the deployer. * * @author bratseth @@ -35,7 +35,7 @@ public class ApplicationMaintainer extends Maintainer { @Override protected void maintain() { Set<ApplicationId> applications = - nodeRepository().getNodes(Node.State.active).stream().map(node -> node.allocation().get().owner()).collect(Collectors.toSet()); + nodeRepository().getNodes(Node.Type.tenant, Node.State.active).stream().map(node -> node.allocation().get().owner()).collect(Collectors.toSet()); for (ApplicationId application : applications) { try { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index 8333996c23e..495ff3b756f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -47,7 +47,7 @@ public abstract class Expirer extends Maintainer { @Override protected void maintain() { List<Node> expired = new ArrayList<>(); - for (Node node : nodeRepository().getNodes(fromState)) { + for (Node node : nodeRepository().getNodes(Node.Type.tenant, fromState)) { Optional<History.Event> event = node.history().event(eventType); if (event.isPresent() && event.get().at().plus(expiryTime).isBefore(clock.instant())) expired.add(node); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index c18aacea284..ba2586a95fd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; -import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceCluster; @@ -25,6 +24,7 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Maintains information in the node repo about when this node last responded to ping @@ -59,17 +59,23 @@ public class NodeFailer extends Maintainer { @Override protected void maintain() { - List<Node> downNodes = maintainDownStatus(); - - for (Node node : downNodes) { - // Grace time before failing the node + for (Node node : determineActiveNodeDownStatus()) { Instant graceTimeEnd = node.history().event(History.Event.Type.down).get().at().plus(downTimeLimit); - if (graceTimeEnd.isBefore(clock.instant()) && ! applicationSuspended(node)) - fail(node); + failActive(node); + } + + for (Node node : readyNodesWithHardwareFailure()) { + nodeRepository().fail(node.hostname()); } } + private List<Node> readyNodesWithHardwareFailure() { + return nodeRepository().getNodes(Node.Type.tenant, Node.State.ready).stream() + .filter(n -> n.status().hardwareFailure()) + .collect(Collectors.toList()); + } + private boolean applicationSuspended(Node node) { try { return orchestrator.getApplicationInstanceStatus(node.allocation().get().owner()) @@ -86,7 +92,7 @@ public class NodeFailer extends Maintainer { * * @return a list of all nodes which are positively currently in the down state */ - private List<Node> maintainDownStatus() { + private List<Node> determineActiveNodeDownStatus() { List<Node> downNodes = new ArrayList<>(); for (ApplicationInstance<ServiceMonitorStatus> application : serviceMonitor.queryStatusOfAllApplicationInstances().values()) { for (ServiceCluster<ServiceMonitorStatus> cluster : application.serviceClusters()) { @@ -133,7 +139,7 @@ public class NodeFailer extends Maintainer { * which is when the node repo has available capacity to replace the node. * Otherwise not replacing the node ensures (by Orchestrator check) that no further action will be taken. */ - private void fail(Node node) { + private void failActive(Node node) { Optional<Deployment> deployment = deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30)); if ( ! deployment.isPresent()) return; // this will be done at another config server @@ -147,7 +153,7 @@ public class NodeFailer extends Maintainer { catch (RuntimeException e) { // The expected reason for deployment to fail here is that there is no capacity available to redeploy. // In that case we should leave the node in the active state to avoid failing additional nodes. - nodeRepository().unfail(node.hostname()); + nodeRepository().reactivate(node.hostname()); log.log(Level.WARNING, "Attempted to fail " + node + " for " + node.allocation().get().owner() + ", but redeploying without the node failed", e); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java index 2753f435bde..b1e0df929f4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetrics.java @@ -50,7 +50,7 @@ public class ProvisionMetrics extends AbstractComponent { log.log(LogLevel.DEBUG, "Running provision metrics task"); try { for (Node.State state : Node.State.values()) - metric.set("hostedVespa." + state.name() + "Hosts", nodeRepository.getNodes(state).size(), null); + metric.set("hostedVespa." + state.name() + "Hosts", nodeRepository.getNodes(Node.Type.tenant, state).size(), null); } catch (RuntimeException e) { log.log(LogLevel.INFO, "Failed gathering metrics data: " + e.getMessage()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java index 3e79acffce3..3d1583eedb5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java @@ -10,7 +10,7 @@ import java.util.stream.Collectors; /** * Filter based on the parent host value (for virtualized nodes). - * @author dybdahl + * @author dybis */ public class ParentHostFilter extends NodeFilter { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java index a605835c11c..e7368e7770c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java @@ -48,5 +48,4 @@ public class StateFilter extends NodeFilter { return new StateFilter(HostFilter.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next); } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 5ff1f41272a..813941de1eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -145,7 +145,8 @@ public class CuratorDatabaseClient { newNodeStatus(node, toState), toState, toState.isAllocated() ? node.allocation() : Optional.empty(), - newNodeHistory(node, toState)); + newNodeHistory(node, toState), + node.type()); curatorTransaction.add(CuratorOperations.delete(toPath(node).getAbsolute())) .add(CuratorOperations.create(toPath(toState, newNode.hostname()).getAbsolute(), nodeSerializer.toJson(newNode))); writtenNodes.add(newNode); @@ -227,13 +228,14 @@ public class CuratorDatabaseClient { private String toDir(Node.State state) { switch (state) { - case provisioned: return "provisioned"; - case ready: return "ready"; - case reserved: return "reserved"; case active: return "allocated"; // legacy name - case inactive: return "deallocated"; // legacy name case dirty: return "dirty"; case failed: return "failed"; + case inactive: return "deallocated"; // legacy name + case parked : return "parked"; + case provisioned: return "provisioned"; + case ready: return "ready"; + case reserved: return "reserved"; default: throw new RuntimeException("Node state " + state + " does not map to a directory name"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 9e0a26be308..00282ccb53d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -55,6 +55,7 @@ public class NodeSerializer { private static final String stateVersionKey = "stateVersion"; private static final String failCountKey = "failCount"; private static final String hardwareFailureKey = "hardwareFailure"; + private static final String nodeTypeKey = "type"; // Configuration fields private static final String flavorKey = "flavor"; @@ -71,7 +72,7 @@ public class NodeSerializer { private static final String dockerImageKey = "dockerImage"; // History event fields - private static final String typeKey = "type"; + private static final String historyEventTypeKey = "type"; private static final String atKey = "at"; private static final String agentKey = "agent"; // retired events only @@ -107,6 +108,7 @@ public class NodeSerializer { object.setBool(hardwareFailureKey, node.status().hardwareFailure()); node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey))); toSlime(node.history(), object.setArray(historyKey)); + object.setString(nodeTypeKey, toString(node.type())); } private void toSlime(Configuration configuration, Cursor object) { @@ -131,7 +133,7 @@ public class NodeSerializer { } private void toSlime(History.Event event, Cursor object) { - object.setString(typeKey, toString(event.type())); + object.setString(historyEventTypeKey, toString(event.type())); object.setLong(atKey, event.at().toEpochMilli()); if (event instanceof History.RetiredEvent) object.setString(agentKey, toString(((History.RetiredEvent)event).agent())); @@ -151,7 +153,8 @@ public class NodeSerializer { statusFromSlime(object), state, allocationFromSlime(object.field(instanceKey)), - historyFromSlime(object.field(historyKey))); + historyFromSlime(object.field(historyKey)), + typeFromSlime(object.field(nodeTypeKey))); } private Status statusFromSlime(Inspector object) { @@ -195,7 +198,7 @@ public class NodeSerializer { } private History.Event eventFromSlime(Inspector object) { - History.Event.Type type = eventTypeFromString(object.field(typeKey).asString()); + History.Event.Type type = eventTypeFromString(object.field(historyEventTypeKey).asString()); if (type == null) return null; Instant at = Instant.ofEpochMilli(object.field(atKey).asLong()); if (type.equals(History.Event.Type.retired)) @@ -225,6 +228,8 @@ public class NodeSerializer { return Optional.empty(); } + // Enum <-> string mappings + /** Returns the event type, or null if this event type should be ignored */ private History.Event.Type eventTypeFromString(String eventTypeString) { switch (eventTypeString) { @@ -239,7 +244,6 @@ public class NodeSerializer { } throw new IllegalArgumentException("Unknown node event type '" + eventTypeString + "'"); } - private String toString(History.Event.Type nodeEventType) { switch (nodeEventType) { case readied : return "readied"; @@ -261,7 +265,6 @@ public class NodeSerializer { } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentString + "'"); } - private String toString(History.RetiredEvent.Agent agent) { switch (agent) { case application : return "application"; @@ -270,4 +273,20 @@ public class NodeSerializer { throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } + private Node.Type typeFromSlime(Inspector object) { + if ( ! object.valid()) return Node.Type.tenant; // TODO: Remove this and change to pass string line when 6.13 is released everywhere + switch (object.asString()) { + case "tenant" : return Node.Type.tenant; + case "host" : return Node.Type.host; + default : throw new IllegalArgumentException("Unknown node type '" + object.asString() + "'"); + } + } + private String toString(Node.Type type) { + switch (type) { + case tenant: return "tenant"; + case host: return "host"; + } + throw new IllegalArgumentException("Unknown node type '" + type.toString() + "'"); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 6b5d37d812a..0f5549e1110 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -78,7 +78,7 @@ class GroupPreparer { // Use new, ready nodes. Need to lock ready pool to ensure that nodes are not grabbed by others. try (Mutex readyLock = nodeRepository.lockUnallocated()) { - List<Node> readyNodes = nodeRepository.getNodes(Node.State.ready); + List<Node> readyNodes = nodeRepository.getNodes(Node.Type.tenant, Node.State.ready); accepted = nodeList.offer(optimize(readyNodes), !canChangeGroup); nodeList.update(nodeRepository.reserve(accepted)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java index 321a75421a2..fa74605b32f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeStateSerializer.java @@ -28,6 +28,7 @@ public class NodeStateSerializer { addMapping(Node.State.dirty, "dirty"); addMapping(Node.State.failed, "failed"); addMapping(Node.State.inactive, "inactive"); + addMapping(Node.State.parked, "parked"); addMapping(Node.State.provisioned, "provisioned"); addMapping(Node.State.ready, "ready"); addMapping(Node.State.reserved, "reserved"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java index f4c52010415..867d45b4fcd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResource.java @@ -47,7 +47,7 @@ public class ProvisionResource { public void addNodes(List<HostInfo> hostInfoList) { List<Node> nodes = new ArrayList<>(); for (HostInfo hostInfo : hostInfoList) - nodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow(hostInfo.flavor)))); + nodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow(hostInfo.flavor)), Node.Type.tenant)); nodeRepository.addNodes(nodes); } @@ -94,7 +94,7 @@ public class ProvisionResource { Map<String, TenantStatus.ApplicationUsage> appinstanceUsageMap = new HashMap<>(); - nodeRepository.getNodes(Node.State.active).stream() + nodeRepository.getNodes(Node.Type.tenant, Node.State.active).stream() .filter(node -> { return node.allocation().get().owner().tenant().value().equals(tenantId); }) @@ -115,11 +115,12 @@ public class ProvisionResource { } //TODO: move this to nodes/v2/ when the spec for this has been nailed. + //TODO: Change it to list host nodes, instead of hosts for tenant nodes. @GET @Path("/dockerhost/{hostname}") public ContainersForHost getContainersForHost(@PathParam("hostname") String hostname) { List<DockerContainer> dockerContainersForHost = - nodeRepository.getNodes(State.active, State.inactive).stream() + nodeRepository.getNodes(Node.Type.tenant, State.active, State.inactive).stream() .filter(runsOnDockerHost(hostname)) .flatMap(ProvisionResource::toDockerContainer) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java index 00e232dcfd3..0c9ca701a6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v1/NodesApiHandler.java @@ -78,13 +78,15 @@ public class } private void toSlime(Node.State state, Cursor object) { - List<Node> nodes = nodeRepository.getNodes(state); Cursor nodeArray = null; // create if there are nodes - for (Node node : nodes) { - if (hostnameFilter.isPresent() && ! node.hostname().equals(hostnameFilter.get())) continue; - if (nodeArray == null) - nodeArray = object.setArray(state.name()); - toSlime(node, nodeArray.addObject()); + for (Node.Type type : Node.Type.values()) { + List<Node> nodes = nodeRepository.getNodes(type, state); + for (Node node : nodes) { + if (hostnameFilter.isPresent() && !node.hostname().equals(hostnameFilter.get())) continue; + if (nodeArray == null) + nodeArray = object.setArray(state.name()); + toSlime(node, nodeArray.addObject()); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 9981602e4d0..b137d10b311 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -14,10 +14,10 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Configuration; +import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; -import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; @@ -44,6 +44,8 @@ public class NodesApiHandler extends LoggingRequestHandler { private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; + private static final String nodeTypeKey = "type"; + public NodesApiHandler(Executor executor, AccessLog accessLog, NodeRepository nodeRepository, NodeFlavors flavors) { super(executor, accessLog); @@ -57,7 +59,7 @@ public class NodesApiHandler extends LoggingRequestHandler { switch (request.getMethod()) { case GET: return handleGET(request); case PUT: return handlePUT(request); - case POST: return handlePOST(request); + case POST: return isPatchOverride(request) ? handlePATCH(request) : handlePOST(request); case DELETE: return handleDELETE(request); case PATCH: return handlePATCH(request); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); @@ -97,12 +99,16 @@ public class NodesApiHandler extends LoggingRequestHandler { nodeRepository.fail(lastElement(path)); return new MessageResponse("Moved " + lastElement(path) + " to failed"); } + else if (path.startsWith("/nodes/v2/state/parked/")) { + nodeRepository.park(lastElement(path)); + return new MessageResponse("Moved " + lastElement(path) + " to parked"); + } else if (path.startsWith("/nodes/v2/state/dirty/")) { nodeRepository.deallocate(lastElement(path)); return new MessageResponse("Moved " + lastElement(path) + " to dirty"); } else if (path.startsWith("/nodes/v2/state/active/")) { - nodeRepository.unfail(lastElement(path)); + nodeRepository.reactivate(lastElement(path)); return new MessageResponse("Moved " + lastElement(path) + " to active"); } else { @@ -189,7 +195,17 @@ public class NodesApiHandler extends LoggingRequestHandler { inspector.field("openStackId").asString(), inspector.field("hostname").asString(), parentHostname, - new Configuration(nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString()))); + new Configuration(nodeFlavors.getFlavorOrThrow(inspector.field("flavor").asString())), + nodeTypeFromSlime(inspector.field(nodeTypeKey))); + } + + private Node.Type nodeTypeFromSlime(Inspector object) { + if (! object.valid()) return Node.Type.tenant; // default + switch (object.asString()) { + case "tenant" : return Node.Type.tenant; + case "host" : return Node.Type.host; + default: throw new IllegalArgumentException("Unknown node type '" + object.asString() + "'"); + } } // TODO: Move most of this to node repo @@ -204,7 +220,12 @@ public class NodesApiHandler extends LoggingRequestHandler { if ( ! node.isPresent()) node = nodeRepository.getNode(Node.State.failed, hostname); if ( ! node.isPresent()) - throw new IllegalArgumentException("Could not set " + hostname + " ready: Not registered as provisioned, dirty or failed"); + node = nodeRepository.getNode(Node.State.parked, hostname); + if ( ! node.isPresent()) + throw new IllegalArgumentException("Could not set " + hostname + " ready: Not registered as provisioned, dirty, failed or parked"); + + if (node.get().allocation().isPresent()) + throw new IllegalArgumentException("Could not set " + hostname + " ready: Node is allocated and must be moved to dirty instead"); nodeRepository.setReady(Collections.singletonList(node.get())); return "Moved " + hostname + " to ready"; @@ -229,4 +250,18 @@ public class NodesApiHandler extends LoggingRequestHandler { return path.substring(lastSlash + 1, path.length()); } + private boolean isPatchOverride(HttpRequest request) { + //Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests. + String override = request.getHeader("X-HTTP-Method-Override"); + if (override != null) { + if (override.equals("PATCH")) { + return true; + } else { + String msg = String.format("Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got '%s'", override); + throw new IllegalArgumentException(msg); + } + } + return false; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 4e8fbe6099b..91fa540e4b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.container.jdisc.HttpRequest; @@ -42,7 +41,7 @@ class NodesResponse extends HttpResponse { private final Slime slime; - public NodesResponse(ResponseType type, HttpRequest request, NodeRepository nodeRepository) { + public NodesResponse(ResponseType responseType, HttpRequest request, NodeRepository nodeRepository) { super(200); this.parentUrl = toParentUrl(request); this.nodeParentUrl = toNodeParentUrl(request); @@ -52,7 +51,7 @@ class NodesResponse extends HttpResponse { slime = new Slime(); Cursor root = slime.setObject(); - switch (type) { + switch (responseType) { case nodeList: nodesToSlime(root); break; case stateList : statesToSlime(root); break; case nodesInStateList: nodesToSlime(stateFromString(lastElement(parentUrl)), root); break; @@ -103,14 +102,17 @@ class NodesResponse extends HttpResponse { /** Outputs the nodes in the given state to a node array */ private void nodesToSlime(Node.State state, Cursor parentObject) { Cursor nodeArray = parentObject.setArray("nodes"); - toSlime(nodeRepository.getNodes(state), nodeArray); + for (Node.Type type : Node.Type.values()) + toSlime(nodeRepository.getNodes(type, state), nodeArray); } /** Outputs all the nodes to a node array */ private void nodesToSlime(Cursor parentObject) { Cursor nodeArray = parentObject.setArray("nodes"); - for (Node.State state : Node.State.values()) - toSlime(nodeRepository.getNodes(state), nodeArray); + for (Node.State state : Node.State.values()) { + for (Node.Type type : Node.Type.values()) + toSlime(nodeRepository.getNodes(type, state), nodeArray); + } } private void toSlime(List<Node> nodes, Cursor array) { @@ -132,7 +134,9 @@ class NodesResponse extends HttpResponse { if ( ! allFields) return; object.setString("id", node.id()); object.setString("state", NodeStateSerializer.wireNameOf(node.state())); + object.setString("type", node.type().name()); object.setString("hostname", node.hostname()); + object.setString("type", toString(node.type())); if (node.parentHostname().isPresent()) { object.setString("parentHostname", node.parentHostname().get()); } @@ -177,6 +181,15 @@ class NodesResponse extends HttpResponse { toSlime(node.history(), object.setArray("history")); } + private String toString(Node.Type type) { + switch(type) { + case tenant: return "tenant"; + case host: return "host"; + default: + throw new RuntimeException("New type added to enum, not implemented in NodesResponse: " + type.name()); + } + } + private void toSlime(ApplicationId id, Cursor object) { object.setString("tenant", id.tenant().value()); object.setString("application", id.application().value()); @@ -211,4 +224,5 @@ class NodesResponse extends HttpResponse { return NodeStateSerializer.fromWireName(stateString) .orElseThrow(() -> new RuntimeException("Node state '" + stateString + "' is not known")); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 990051b2317..3ca0c4ff365 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.hosted.provision.testutils; * For running NodeRepository API with some mocked data. * This is used by both NodeAdmin and NodeRepository tests. * - * @author dybdahl + * @author dybis */ public class ContainerConfig { public static final String servicesXmlV2(int port) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index d60e43cebed..123104a354f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -48,22 +48,22 @@ public class MockNodeRepository extends NodeRepository { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); List<Node> nodes = new ArrayList<>(); - nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("expensive")))); + nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("expensive")), Node.Type.tenant)); // TODO: Use docker flavor - Node node4 = createNode("node4", "host4.yahoo.com", Optional.of("dockerhost4"), new Configuration(flavors.getFlavorOrThrow("default"))); + Node node4 = createNode("node4", "host4.yahoo.com", Optional.of("dockerhost4"), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant); node4 = node4.setStatus(node4.status().setDockerImage("image-12")); nodes.add(node4); - Node node5 = createNode("node5", "host5.yahoo.com", Optional.of("dockerhost"), new Configuration(flavors.getFlavorOrThrow("default"))); + Node node5 = createNode("node5", "host5.yahoo.com", Optional.of("dockerhost"), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant); nodes.add(node5.setStatus(node5.status().setDockerImage("image-123"))); - nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node7", "host7.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node7", "host7.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); // 8 and 9 are added by web service calls - Node node10 = createNode("node10", "host10.yahoo.com", Optional.of("parent.yahoo.com"), new Configuration(flavors.getFlavorOrThrow("default"))); + Node node10 = createNode("node10", "host10.yahoo.com", Optional.of("parent.yahoo.com"), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant); Status node10newStatus = node10.status(); node10newStatus = node10newStatus .setVespaVersion(Version.fromString("5.104.142")) @@ -71,6 +71,9 @@ public class MockNodeRepository extends NodeRepository { .setStateVersion("5.104.142-2.1.2408"); node10 = node10.setStatus(node10newStatus); nodes.add(node10); + + nodes.add(createNode("parent1", "parent1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.host)); + nodes = addNodes(nodes); nodes.remove(6); setReady(nodes); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java index 99c6c7d9294..e034b185a02 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainerTest.java @@ -49,53 +49,63 @@ public class ApplicationMaintainerTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock); createReadyNodes(15, nodeRepository, nodeFlavors); + createHostNodes(2, nodeRepository, nodeFlavors); Fixture fixture = new Fixture(zone, nodeRepository, nodeFlavors); // Create applications fixture.activate(); - // Fail some nodes + // Fail and park some nodes nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname()); nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(0).hostname()); - nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(4).hostname()); + nodeRepository.park(nodeRepository.getNodes(fixture.app2).get(4).hostname()); int failedInApp1 = 1; - int failedInApp2 = 2; + int failedOrParkedInApp2 = 2; assertEquals(fixture.wantedNodesApp1 - failedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); - assertEquals(fixture.wantedNodesApp2 - failedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); - assertEquals(failedInApp1 + failedInApp2, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals(3, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(fixture.wantedNodesApp2 - failedOrParkedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); + assertEquals(failedInApp1 + failedOrParkedInApp2, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed, Node.State.parked).size()); + assertEquals(3, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.host, Node.State.ready).size()); // Cause maintenance deployment which will allocate replacement nodes fixture.runApplicationMaintainer(); assertEquals(fixture.wantedNodesApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); assertEquals(fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); - assertEquals(0, nodeRepository.getNodes(Node.State.ready).size()); - - // Unfail the previously failed nodes - nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); - nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); - nodeRepository.unfail(nodeRepository.getNodes(Node.State.failed).get(0).hostname()); - int unfailedInApp1 = 1; - int unfailedInApp2 = 2; - assertEquals(0, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals(fixture.wantedNodesApp1 + unfailedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); - assertEquals(fixture.wantedNodesApp2 + unfailedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); - assertEquals("The unfailed nodes are now active but not part of the application", + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + + // Reactivate the previously failed nodes + nodeRepository.reactivate(nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).get(0).hostname()); + nodeRepository.reactivate(nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).get(0).hostname()); + nodeRepository.reactivate(nodeRepository.getNodes(Node.Type.tenant, Node.State.parked).get(0).hostname()); + int reactivatedInApp1 = 1; + int reactivatedInApp2 = 2; + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals(fixture.wantedNodesApp1 + reactivatedInApp1, nodeRepository.getNodes(fixture.app1, Node.State.active).size()); + assertEquals(fixture.wantedNodesApp2 + reactivatedInApp2, nodeRepository.getNodes(fixture.app2, Node.State.active).size()); + assertEquals("The reactivated nodes are now active but not part of the application", 0, fixture.getNodes(Node.State.active).retired().size()); // Cause maintenance deployment which will update the applications with the re-activated nodes fixture.runApplicationMaintainer(); assertEquals("Superflous content nodes are retired", - unfailedInApp2, fixture.getNodes(Node.State.active).retired().size()); + reactivatedInApp2, fixture.getNodes(Node.State.active).retired().size()); assertEquals("Superflous container nodes are deactivated (this makes little point for container nodes)", - unfailedInApp1, fixture.getNodes(Node.State.inactive).size()); + reactivatedInApp1, fixture.getNodes(Node.State.inactive).size()); } private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { List<Node> nodes = new ArrayList<>(count); for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + + private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) + nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); nodes = nodeRepository.addNodes(nodes); nodeRepository.setReady(nodes); } @@ -140,7 +150,7 @@ public class ApplicationMaintainerTest { } NodeList getNodes(Node.State ... states) { - return new NodeList(nodeRepository.getNodes(states)); + return new NodeList(nodeRepository.getNodes(Node.Type.tenant, states)); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 278a9b704b9..a1d9268ee33 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -45,18 +45,18 @@ public class FailedExpirerTest { public void ensure_failed_nodes_are_deallocated_in_prod() throws InterruptedException { NodeRepository nodeRepository = failureScenarioIn(Environment.prod); - assertEquals(2, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals(1, nodeRepository.getNodes(Node.State.dirty).size()); - assertEquals("node3", nodeRepository.getNodes(Node.State.dirty).get(0).hostname()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals(1, nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).size()); + assertEquals("node3", nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).get(0).hostname()); } @Test public void ensure_failed_nodes_are_deallocated_in_dev() throws InterruptedException { NodeRepository nodeRepository = failureScenarioIn(Environment.dev); - assertEquals(1, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size()); - assertEquals("node2", nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + assertEquals(1, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).size()); + assertEquals("node2", nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).get(0).hostname()); } private NodeRepository failureScenarioIn(Environment environment) { @@ -66,11 +66,17 @@ public class FailedExpirerTest { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); List<Node> nodes = new ArrayList<>(3); - nodes.add(nodeRepository.createNode("node1", "node1", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); - nodes.add(nodeRepository.createNode("node2", "node2", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); - nodes.add(nodeRepository.createNode("node3", "node3", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node1", "node1", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(nodeRepository.createNode("node2", "node2", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(nodeRepository.createNode("node3", "node3", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); nodeRepository.addNodes(nodes); + List<Node> hostNodes = new ArrayList<>(1); + hostNodes.add(nodeRepository.createNode("parent1", "parent1", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); + hostNodes.add(nodeRepository.createNode("parent2", "parent2", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); + nodeRepository.addNodes(hostNodes); + + // Set node1 to have failed 4 times before Node node1 = nodeRepository.getNode("node1").get(); node1 = node1.setStatus(node1.status().increaseFailCount()); @@ -85,20 +91,20 @@ public class FailedExpirerTest { nodeRepository.write(node2); // Allocate the nodes - nodeRepository.setReady(nodeRepository.getNodes(Node.State.provisioned)); + nodeRepository.setReady(nodeRepository.getNodes(Node.Type.tenant, Node.State.provisioned)); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(3), 1, null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, asHosts(nodes)); transaction.commit(); - assertEquals(3, nodeRepository.getNodes(Node.State.active).size()); + assertEquals(3, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); // Fail the nodes nodeRepository.fail("node1"); nodeRepository.fail("node2"); nodeRepository.fail("node3"); - assertEquals(3, nodeRepository.getNodes(Node.State.failed).size()); + assertEquals(3, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); // Failure times out clock.advance(Duration.ofDays(5)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 460ab3906ed..0c853e4b1dd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -50,10 +50,15 @@ public class InactiveAndFailedExpirerTest { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, Zone.defaultZone(), clock); List<Node> nodes = new ArrayList<>(2); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); nodeRepository.addNodes(nodes); + List<Node> hostNodes = new ArrayList<>(2); + hostNodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); + hostNodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); + nodeRepository.addNodes(hostNodes); + // Allocate then deallocate 2 nodes nodeRepository.setReady(nodes); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -62,16 +67,16 @@ public class InactiveAndFailedExpirerTest { NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, asHosts(nodes)); transaction.commit(); - assertEquals(2, nodeRepository.getNodes(Node.State.active).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); nodeRepository.deactivate(applicationId); - assertEquals(2, nodeRepository.getNodes(Node.State.inactive).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.inactive).size()); // Inactive times out clock.advance(Duration.ofMinutes(14)); new InactiveExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); - assertEquals(0, nodeRepository.getNodes(Node.State.inactive).size()); - List<Node> dirty = nodeRepository.getNodes(Node.State.dirty); + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.inactive).size()); + List<Node> dirty = nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty); assertEquals(2, dirty.size()); assertFalse(dirty.get(0).allocation().isPresent()); assertFalse(dirty.get(1).allocation().isPresent()); @@ -84,8 +89,8 @@ public class InactiveAndFailedExpirerTest { // Dirty times out for the other one clock.advance(Duration.ofMinutes(14)); new DirtyExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); - assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size()); - List<Node> failed = nodeRepository.getNodes(Node.State.failed); + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).size()); + List<Node> failed = nodeRepository.getNodes(Node.Type.tenant, Node.State.failed); assertEquals(1, failed.size()); assertEquals(1, failed.get(0).status().failCount()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 39a72ef16e8..c568b9db2a7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -89,7 +89,8 @@ public class NodeFailerTest { nodeRepository = new NodeRepository(NODE_FLAVORS, curator, clock); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, NODE_FLAVORS, ZONE); - createReadyNodes(14, nodeRepository, NODE_FLAVORS); + createReadyNodes(16, nodeRepository, NODE_FLAVORS); + createHostNodes(3, nodeRepository, NODE_FLAVORS); // Create applications ClusterSpec clusterApp1 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Optional.empty()); @@ -139,11 +140,22 @@ public class NodeFailerTest { failer.run(); clock.advance(Duration.ofMinutes(5)); assertEquals( 0, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 0, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 4, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); } + // Failures are detected on two ready nodes, which are then failed + Node readyFail1 = nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).get(2); + Node readyFail2 = nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).get(3); + nodeRepository.write(readyFail1.setStatus(readyFail1.status().setHardwareFailure(true))); + nodeRepository.write(readyFail2.setStatus(readyFail2.status().setHardwareFailure(true))); + assertEquals(4, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + failer.run(); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + assertEquals(Node.State.failed, nodeRepository.getNode(readyFail1.hostname()).get().state()); + assertEquals(Node.State.failed, nodeRepository.getNode(readyFail2.hostname()).get().state()); + String downHost1 = nodeRepository.getNodes(APP_1, Node.State.active).get(1).hostname(); String downHost2 = nodeRepository.getNodes(APP_2, Node.State.active).get(3).hostname(); serviceMonitor.setHostDown(downHost1); @@ -153,9 +165,9 @@ public class NodeFailerTest { failer.run(); clock.advance(Duration.ofMinutes(5)); assertEquals( 0, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 0, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 2, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 2, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 2, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); } serviceMonitor.setHostUp(downHost1); for (int minutes = 0; minutes < 30; minutes +=5 ) { @@ -165,10 +177,10 @@ public class NodeFailerTest { // downHost2 should now be failed and replaced, but not downHost1 assertEquals( 1, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size()); - assertEquals(downHost2, nodeRepository.getNodes(Node.State.failed).get(0).hostname()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 3, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 1, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + assertEquals(downHost2, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).get(0).hostname()); // downHost1 fails again serviceMonitor.setHostDown(downHost1); @@ -180,17 +192,17 @@ public class NodeFailerTest { failer.run(); // due to this, nothing is failed assertEquals( 1, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 1, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 1, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 3, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 1, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); // when status becomes known, and the host is still down, it is failed clock.advance(Duration.ofMinutes(5)); serviceMonitor.setStatusIsKnown(true); failer.run(); assertEquals( 2, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 4, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); // the last host goes down Node lastNode = highestIndex(nodeRepository.getNodes(APP_1, Node.State.active)); @@ -200,19 +212,19 @@ public class NodeFailerTest { failer.run(); clock.advance(Duration.ofMinutes(5)); assertEquals( 2, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 2, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 4, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); } // A new node is available - createReadyNodes(1, 14, nodeRepository, NODE_FLAVORS); + createReadyNodes(1, 16, nodeRepository, NODE_FLAVORS); failer.run(); // The node is now failed assertEquals( 3, deployer.redeployments); - assertEquals(12, nodeRepository.getNodes(Node.State.active).size()); - assertEquals( 3, nodeRepository.getNodes(Node.State.failed).size()); - assertEquals( 0, nodeRepository.getNodes(Node.State.ready).size()); + assertEquals(12, nodeRepository.getNodes(Node.Type.tenant, Node.State.active).size()); + assertEquals( 5, nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).size()); + assertEquals( 0, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); assertTrue("The index of the last failed node is not reused", highestIndex(nodeRepository.getNodes(APP_1, Node.State.active)).allocation().get().membership().index() > @@ -226,7 +238,15 @@ public class NodeFailerTest { private void createReadyNodes(int count, int startIndex, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < startIndex + count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + + private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) + nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); nodes = nodeRepository.addNodes(nodes); nodeRepository.setReady(nodes); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 10e685e3f96..8c0563e4bc8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -41,25 +41,26 @@ public class ReservationExpirerTest { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, flavors, Zone.defaultZone(), clock); List<Node> nodes = new ArrayList<>(2); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.host)); nodes = nodeRepository.addNodes(nodes); // Reserve 2 nodes - assertEquals(2, nodeRepository.getNodes(Node.State.provisioned).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.provisioned).size()); nodeRepository.setReady(nodes); ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1, null); - assertEquals(2, nodeRepository.getNodes(Node.State.reserved).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.reserved).size()); // Reservation times out clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); // Assert nothing is reserved - assertEquals(0, nodeRepository.getNodes(Node.State.reserved).size()); - List<Node> dirty = nodeRepository.getNodes(Node.State.dirty); + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.reserved).size()); + List<Node> dirty = nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty); assertEquals(2, dirty.size()); assertFalse(dirty.get(0).allocation().isPresent()); assertFalse(dirty.get(1).allocation().isPresent()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index f6f26aeced6..be80690a972 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -49,6 +49,7 @@ public class RetiredExpirerTest { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); createReadyNodes(7, nodeRepository, nodeFlavors); + createHostNodes(4, nodeRepository, nodeFlavors); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -86,6 +87,7 @@ public class RetiredExpirerTest { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); createReadyNodes(8, nodeRepository, nodeFlavors); + createHostNodes(4, nodeRepository, nodeFlavors); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -120,7 +122,15 @@ public class RetiredExpirerTest { private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { List<Node> nodes = new ArrayList<>(count); for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes = nodeRepository.addNodes(nodes); + nodeRepository.setReady(nodes); + } + + private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { + List<Node> nodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) + nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host)); nodes = nodeRepository.addNodes(nodes); nodeRepository.setReady(nodes); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java index d2092e5be13..0fe507edbe5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/monitoring/ProvisionMetricsTest.java @@ -28,11 +28,14 @@ public class ProvisionMetricsTest { final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); final Curator curator = new MockCurator(); final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator); - final Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + final Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant); nodeRepository.addNodes(Collections.singletonList(node)); + final Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host); + nodeRepository.addNodes(Collections.singletonList(hostNode)); final Map<String, Number> expectedMetrics = new HashMap<>(); expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); expectedMetrics.put("hostedVespa.activeHosts", 0); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index 50d0e56d999..7f80a83bef7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -24,7 +24,7 @@ public class CuratorDatabaseClientTest { private CuratorDatabaseClient zkClient = new CuratorDatabaseClient(FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC()); @Test - public void ensure_can_read_stored_host_with_instance_information() throws Exception { + public void ensure_can_read_stored_host_with_instance_information_no_type() throws Exception { String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159,\"allocated\":1421057746687,\"instance\":{\"tenantId\":\"by_mortent\",\"applicationId\":\"music\",\"instanceId\":\"default\",\"serviceId\":\"container/default/0/0\"}}"; curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/allocated/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); @@ -32,15 +32,17 @@ public class CuratorDatabaseClientTest { List<Node> allocatedNodes = zkClient.getNodes(Node.State.active); assertEquals(1, allocatedNodes.size()); assertEquals("container/default/0/0", allocatedNodes.get(0).allocation().get().membership().stringValue()); + assertEquals(Node.Type.tenant, allocatedNodes.get(0).type()); } @Test public void ensure_can_read_stored_host_information() throws Exception { - String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159}"; + String zkline = "{\"hostname\":\"oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com\",\"openStackId\":\"7951bb9d-3989-4a60-a21c-13690637c8ea\",\"configuration\":{\"flavor\":\"default\"},\"created\":1421054425159, \"type\":\"host\"}"; curator.framework().create().creatingParentsIfNeeded().forPath("/provision/v1/ready/oxy-oxygen-0a4ae4f1.corp.bf1.yahoo.com", zkline.getBytes()); List<Node> allocatedNodes = zkClient.getNodes(Node.State.ready); assertEquals(1, allocatedNodes.size()); + assertEquals(Node.Type.host, allocatedNodes.get(0).type()); } /** Test that locks can be acquired and released */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 7dc52148e8b..9ba29cec588 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -65,6 +65,7 @@ public class SerializationTest { node = node.setStatus(node.status().setVespaVersion(Version.fromString("1.2.3"))); node = node.setStatus(node.status().increaseFailCount().increaseFailCount()); node = node.setStatus(node.status().setHardwareFailure(true)); + node = node.setType(Node.Type.tenant); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); assertEquals(node.id(), copy.id()); @@ -83,10 +84,22 @@ public class SerializationTest { assertEquals(node.allocation().get().removable(), copy.allocation().get().removable()); assertEquals(1, copy.history().events().size()); assertEquals(clock.instant(), copy.history().event(History.Event.Type.reserved).get().at()); + assertEquals(Node.Type.tenant, copy.type()); } @Test - public void testRebootAndRestartNoCurrentValuesSerialization() { + public void testDefaultType() { + Node node = createNode().allocate(ApplicationId.from(TenantName.from("myTenant"), + ApplicationName.from("myApplication"), + InstanceName.from("myInstance")), + ClusterMembership.from("content/myId/0/0", Optional.empty()), + clock.instant()); + Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); + assertEquals(Node.Type.host, copy.type()); + } + + @Test + public void testRebootAndRestartandTypeNoCurrentValuesSerialization() { String nodeData = "{\n" + " \"rebootGeneration\" : 0,\n" + " \"configuration\" : {\n" + @@ -116,6 +129,7 @@ public class SerializationTest { assertEquals(0, node.status().reboot().current()); assertEquals(0, node.allocation().get().restartGeneration().wanted()); assertEquals(0, node.allocation().get().restartGeneration().current()); + assertEquals(Node.Type.tenant, node.type()); } @Test @@ -194,7 +208,7 @@ public class SerializationTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", "myHostname", Optional.of(parentHostname), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + Node node = Node.create("myId", "myHostname", Optional.of(parentHostname), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); @@ -234,7 +248,7 @@ public class SerializationTest { } private Node createNode() { - return Node.create("myId", "myHostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default"))); + return Node.create("myId", "myHostname", Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.host); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 01d8a0eeabf..8eec59590c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -166,7 +166,7 @@ public class ProvisioningTester implements AutoCloseable { public void fail(HostSpec host) { int beforeFailCount = nodeRepository.getNode(Node.State.active, host.hostname()).get().status().failCount(); Node failedNode = nodeRepository.fail(host.hostname()); - assertTrue(nodeRepository.getNodes(Node.State.failed).contains(failedNode)); + assertTrue(nodeRepository.getNodes(Node.Type.tenant, Node.State.failed).contains(failedNode)); assertEquals(beforeFailCount + 1, failedNode.status().failCount()); } @@ -205,7 +205,7 @@ public class ProvisioningTester implements AutoCloseable { nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), - new Configuration(nodeFlavors.getFlavorOrThrow(flavor)))); + new Configuration(nodeFlavors.getFlavorOrThrow(flavor)), Node.Type.tenant)); nodes = nodeRepository.addNodes(nodes); nodeRepository.setReady(nodes); return nodes; @@ -222,7 +222,7 @@ public class ProvisioningTester implements AutoCloseable { for (int i = 0; i < n; i++) { final String hostname = UUID.randomUUID().toString(); nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId, - new Configuration(nodeFlavors.getFlavorOrThrow(flavor)))); + new Configuration(nodeFlavors.getFlavorOrThrow(flavor)), Node.Type.tenant)); } nodes = nodeRepository.addNodes(nodes); nodeRepository.setReady(nodes); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java index 56c2e755c21..4acd748c3d0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/legacy/ProvisionResourceTest.java @@ -56,14 +56,14 @@ public class ProvisionResourceTest { List<Node> readyNodes = new ArrayList<>(); for (HostInfo hostInfo : createHostInfos(readyCount, 0)) readyNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, - Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); readyNodes = nodeRepository.addNodes(readyNodes); nodeRepository.setReady(readyNodes); List<Node> provisionedNodes = new ArrayList<>(); for (HostInfo hostInfo : createHostInfos(provisionedCount, readyCount)) provisionedNodes.add(nodeRepository.createNode(hostInfo.openStackId, hostInfo.hostname, - Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")))); + Optional.empty(), new Configuration(nodeFlavors.getFlavorOrThrow("default")), Node.Type.tenant)); nodeRepository.addNodes(provisionedNodes); } @@ -111,12 +111,12 @@ public class ProvisionResourceTest { assignNode(application, 2); nodeRepository.deactivate(application); List<Node> nodes = nodeRepository.deallocate(nodeRepository.getNodes(application, Node.State.inactive)); - assertEquals(0, nodeRepository.getNodes(Node.State.ready).size()); - assertEquals(2, nodeRepository.getNodes(Node.State.dirty).size()); + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).size()); provisionResource.setReady(nodes.get(0).hostname()); provisionResource.setReady(nodes.get(1).hostname()); - assertEquals(2, nodeRepository.getNodes(Node.State.ready).size()); - assertEquals(0, nodeRepository.getNodes(Node.State.dirty).size()); + assertEquals(2, nodeRepository.getNodes(Node.Type.tenant, Node.State.ready).size()); + assertEquals(0, nodeRepository.getNodes(Node.Type.tenant, Node.State.dirty).size()); } @Test(expected = IllegalArgumentException.class) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java index 9afb14f632a..fe027bddae0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v1/RestApiTest.java @@ -85,12 +85,12 @@ public class RestApiTest { NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); List<Node> nodes = new ArrayList<>(); - nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node4", "host4.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node5", "host5.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); - nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")))); + nodes.add(createNode("node1", "host1.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node2", "host2.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node3", "host3.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node4", "host4.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node5", "host5.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); + nodes.add(createNode("node6", "host6.yahoo.com", Optional.empty(), new Configuration(flavors.getFlavorOrThrow("default")), Node.Type.tenant)); nodes = addNodes(nodes); nodes.remove(5); setReady(nodes); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 922e0038ea5..3992e808830 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -60,7 +60,7 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertReboot(8, new Request("http://localhost:8080/nodes/v2/command/reboot", + assertReboot(9, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host3.yahoo.com"), "\"rebootGeneration\":3"); @@ -69,13 +69,15 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default") + "," + asNodeJson("host9.yahoo.com", "large-variant") + "," + + asHostJson("parent2.yahoo.com", "large-variant") + "," + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), - "{\"message\":\"Added 3 nodes to the provisioned state\"}"); + "{\"message\":\"Added 4 nodes to the provisioned state\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // PUT nodes ready assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", @@ -99,12 +101,12 @@ public class RestApiTest { new byte[0], Request.Method.PUT), "{\"message\":\"Moved host8.yahoo.com to active\"}"); - // PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host8.yahoo.com", + // PUT a node in parked ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host8.yahoo.com to failed\"}"); + "{\"message\":\"Moved host8.yahoo.com to parked\"}"); assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"failed\""); + "\"state\":\"parked\""); // ... and delete it assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", new byte[0], Request.Method.DELETE), @@ -151,7 +153,53 @@ public class RestApiTest { } @Test + public void post_with_patch_method_override_in_header_is_handled_as_patch() throws IOException { + Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); + req.getHeaders().add("X-HTTP-Method-Override", "PATCH"); + assertResponse(req, "{\"message\":\"Updated host4.yahoo.com\"}"); + } + + @Test + public void post_with_invalid_method_override_in_header_gives_sane_error_message() throws IOException { + Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); + req.getHeaders().add("X-HTTP-Method-Override", "GET"); + assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); + } + + @Test public void testInvalidRequests() throws IOException { + // Attempt to fail and ready an allocated node without going through dirty + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to failed\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set host1.yahoo.com ready: Node is allocated and must be moved to dirty instead\"}"); + // (... while dirty then ready works (the ready move will be initiated by node maintenance)) + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to ready\"}"); + + // Attempt to park and ready an allocated node without going through dirty + assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to parked\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set host2.yahoo.com ready: Node is allocated and must be moved to dirty instead\"}"); + // (... while dirty then ready works (the ready move will be initiated by node maintenance)) + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to ready\"}"); + // Attempt to DELETE a node which is not put in failed first assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", new byte[0], Request.Method.DELETE), @@ -216,6 +264,12 @@ public class RestApiTest { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"" + flavor + "\"}"; } + private String asHostJson(String hostname, String flavor) { + return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\",\"flavor\":\"" + flavor + "\"" + + ", \"type\":\"host\"}"; + } + + /** Asserts a particular response and 200 as response status */ private void assertResponse(Request request, String responseMessage) throws IOException { assertResponse(request, 200, responseMessage); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/host1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/host1.json new file mode 100644 index 00000000000..4847cf1fa9e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/host1.json @@ -0,0 +1,34 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/parent1.yahoo.com", + "id": "parent1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "parent1.yahoo.com", + "openStackId": "what", + "flavor": "default", + "minDiskAvailableGb":400.0, + "minMainMemoryAvailableGb":16.0, + "description":"Host node", + "minCpuCores":2.0, + "canonicalFlavor": "default", + "environment":"env", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure" : false, + "history":[{"event":"readied","at":123},{"event":"reserved","at":123},{"event":"activated","at":123}] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index 734b6702c1e..eeb86038e52 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com", "id": "host1.yahoo.com", "state": "active", + "type": "tenant", "hostname": "host1.yahoo.com", "openStackId": "node1", "flavor": "default", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index d412e803bf5..d9efb50af66 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com", "id": "host10.yahoo.com", "state": "reserved", + "type": "tenant", "hostname": "host10.yahoo.com", "parentHostname": "parent.yahoo.com", "openStackId": "node10", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json index 6d1922e7fc0..e74d95daf89 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -2,6 +2,7 @@ "url":"http://localhost:8080/nodes/v2/node/host11.yahoo.com", "id":"host11.yahoo.com", "state":"provisioned", + "type": "tenant", "hostname":"host11.yahoo.com", "parentHostname":"parent.host.yahoo.com", "openStackId":"host11.yahoo.com", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 830c866ae81..dddba8ccc44 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com", "id": "host2.yahoo.com", "state": "active", + "type": "tenant", "hostname": "host2.yahoo.com", "openStackId": "node2", "flavor": "default", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json index 5bf8631797a..5f50d4b9bb1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com", "id": "host3.yahoo.com", "state": "active", + "type": "tenant", "hostname": "host3.yahoo.com", "openStackId": "node3", "flavor":"expensive", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index ef88154fc5c..80b06a8056e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", "state": "reserved", + "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "parent.yahoo.com", "openStackId": "node4", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index a1bc67705ce..01d4b1b65d8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", "id": "host4.yahoo.com", "state": "reserved", + "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname":"dockerhost4", "openStackId": "node4", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index da4b49280c7..ce4ffc3ec86 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", "id": "host5.yahoo.com", "state": "failed", + "type": "tenant", "hostname": "host5.yahoo.com", "parentHostname":"dockerhost", "openStackId": "node5", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 9ef8adb1f07..378347d9ac2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", "id": "host6.yahoo.com", "state": "active", + "type": "tenant", "hostname": "host6.yahoo.com", "openStackId": "node6", "flavor": "default", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json index 52f01407b2b..855450b640a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com", "id": "host7.yahoo.com", "state": "provisioned", + "type": "tenant", "hostname": "host7.yahoo.com", "openStackId": "node7", "flavor": "default", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index c00b6ed797c..0060c315171 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host8.yahoo.com", "id": "host8.yahoo.com", "state": "provisioned", + "type": "tenant", "hostname": "host8.yahoo.com", "openStackId": "host8.yahoo.com", "flavor": "default", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index 73a0eb8a266..70fa6423fe4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -2,6 +2,7 @@ "url": "http://localhost:8080/nodes/v2/node/host9.yahoo.com", "id": "host9.yahoo.com", "state": "provisioned", + "type": "tenant", "hostname": "host9.yahoo.com", "openStackId": "host9.yahoo.com", "flavor": "large-variant", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json index 8ea48599f0e..ac9b247f8e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -1,6 +1,7 @@ { "nodes": [ @include(node7.json), + @include(parent1.json), @include(node10.json), @include(node4.json), @include(node6.json), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json index 73947ded547..d9f0d9ed34a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -4,6 +4,9 @@ "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" }, { + "url":"http://localhost:8080/nodes/v2/node/parent1.yahoo.com" + }, + { "url":"http://localhost:8080/nodes/v2/node/host10.yahoo.com" }, { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json index 28a17b03cc6..7352c4f4455 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent-nodes.json @@ -1,5 +1,5 @@ { "nodes": [ @include(node10.json) -] + ] }
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json new file mode 100644 index 00000000000..c2d663fcb15 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json @@ -0,0 +1,25 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/parent1.yahoo.com", + "id": "parent1.yahoo.com", + "state": "ready", + "type": "host", + "hostname": "parent1.yahoo.com", + "openStackId": "parent1", + "flavor": "default", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "canonicalFlavor": "default", + "environment": "env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "history": [ + { + "event": "readied", + "at": 123 + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json new file mode 100644 index 00000000000..9b9f1179c1c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -0,0 +1,20 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/parent2.yahoo.com", + "id": "parent2.yahoo.com", + "state": "provisioned", + "type": "host", + "hostname": "parent2.yahoo.com", + "openStackId": "parent2.yahoo.com", + "flavor": "large-variant", + "minDiskAvailableGb": 2000.0, + "minMainMemoryAvailableGb": 128.0, + "description": "Flavor-name-is-large-variant", + "minCpuCores": 64.0, + "canonicalFlavor": "large", + "environment": "env", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "history": [] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json index b83616d2e0b..02d8e473f27 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -9,6 +9,7 @@ "ready": { "url": "http://localhost:8080/nodes/v2/state/ready", "nodes": [ + @include(parent1.json) ] }, "reserved": { @@ -42,6 +43,11 @@ "nodes": [ @include(node5.json) ] + }, + "parked": { + "url": "http://localhost:8080/nodes/v2/state/parked", + "nodes": [ + ] } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json index b2d7354a6c9..4b2de7532dd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json @@ -20,6 +20,9 @@ }, "failed": { "url": "http://localhost:8080/nodes/v2/state/failed" + }, + "parked": { + "url": "http://localhost:8080/nodes/v2/state/parked" } } }
\ No newline at end of file diff --git a/orchestrator-restapi/OWNERS b/orchestrator-restapi/OWNERS index 9ecc8472a21..f44aaadea3c 100644 --- a/orchestrator-restapi/OWNERS +++ b/orchestrator-restapi/OWNERS @@ -1,2 +1,2 @@ bakksjo -hakon +hakonhall diff --git a/orchestrator-restapi/pom.xml b/orchestrator-restapi/pom.xml index 58480cf0423..b2b581a7fc6 100644 --- a/orchestrator-restapi/pom.xml +++ b/orchestrator-restapi/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator-restapi</artifactId> <version>6-SNAPSHOT</version> diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/ApplicationSuspensionApi.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/ApplicationSuspensionApi.java index e40e06062c2..ec64781b213 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/ApplicationSuspensionApi.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/ApplicationSuspensionApi.java @@ -16,7 +16,7 @@ import java.util.Set; * Implementing classes must not put any JAX-RS annotation on the overridden methods. Doing so will cause all * method annotations in this interface to be ignored by the JAX-RS container (see section 3.6 of JSR-339). * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public interface ApplicationSuspensionApi { /** diff --git a/orchestrator/OWNERS b/orchestrator/OWNERS index 3d08c49311b..e131dacde49 100644 --- a/orchestrator/OWNERS +++ b/orchestrator/OWNERS @@ -1 +1 @@ -hakon +hakonhall diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 7ad5393aa31..7647266c374 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator</artifactId> <version>6-SNAPSHOT</version> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationIdNotFoundException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationIdNotFoundException.java index 0e4535a3a51..d12bd66bc86 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationIdNotFoundException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationIdNotFoundException.java @@ -4,7 +4,15 @@ package com.yahoo.vespa.orchestrator; /** * Thrown when applicationId is invalid or not found. * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class ApplicationIdNotFoundException extends Exception { + + public ApplicationIdNotFoundException() { + super(); + } + + public ApplicationIdNotFoundException(String reason) { + super(reason); + } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationStateChangeDeniedException.java index 28f92b3dca2..0ed379e5ece 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationStateChangeDeniedException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ApplicationStateChangeDeniedException.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.orchestrator; * Exception covering all cases where the state change could not * be executed. * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class ApplicationStateChangeDeniedException extends Exception { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/HostNameNotFoundException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/HostNameNotFoundException.java index 5919f974234..9a1b5c5b6b0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/HostNameNotFoundException.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/HostNameNotFoundException.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.applicationmodel.HostName; /** * Exception thrown if hostname is not found in the system (i.e node repo) * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class HostNameNotFoundException extends OrchestrationException { public HostNameNotFoundException(HostName hostName) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index a7db8e42e46..0adafaaa628 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -26,7 +26,7 @@ import java.util.Set; * host level state and the host level policies. * This is used for parallel upgrade and larger maintenance tasks. * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public interface Orchestrator { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index ad72a43ff72..7197c9bcd43 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -36,7 +36,7 @@ import java.util.stream.Collectors; /** * @author oyving - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class OrchestratorImpl implements Orchestrator { @@ -136,7 +136,7 @@ public class OrchestratorImpl implements Orchestrator { @Override public ApplicationInstanceStatus getApplicationInstanceStatus( final ApplicationId appId) throws ApplicationIdNotFoundException { - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId); + ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId,instanceLookupService); return statusService.forApplicationInstance(appRef).getApplicationInstanceStatus(); } @@ -271,7 +271,8 @@ public class OrchestratorImpl implements Orchestrator { final ApplicationId appId, final ApplicationInstanceStatus status) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException{ - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId); + + ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService); try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly(appRef)) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java index ca7bd058169..fff69cfd17b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; @@ -14,8 +12,11 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -108,37 +109,38 @@ public class OrchestratorUtil { } - public static ApplicationInstanceReference toApplicationInstanceReference(ApplicationId appId) { - TenantId tenantId = new TenantId(appId.tenant().toString()); + public static ApplicationInstanceReference toApplicationInstanceReference(ApplicationId appId, + InstanceLookupService instanceLookupService) + throws ApplicationIdNotFoundException { + + Set<ApplicationInstanceReference> appRefs = instanceLookupService.knownInstances(); + List<ApplicationInstanceReference> appRefList = appRefs.stream() + .filter(a -> OrchestratorUtil.toApplicationId(a).equals(appId)) + .collect(Collectors.toList()); - String appName = appId.application().toString(); - String instanceName = appId.instance().toString(); - ApplicationInstanceId appInstanceId = new ApplicationInstanceId(appName + ":" + instanceName); + if (appRefList.size() > 1) { + String msg = String.format("ApplicationId '%s' was not unique but mapped to '%s'", appId, appRefList); + throw new ApplicationIdNotFoundException(msg); + } - return new ApplicationInstanceReference(tenantId,appInstanceId); + if (appRefList.size() == 0) { + throw new ApplicationIdNotFoundException(); + } + + return appRefList.get(0); } public static ApplicationId toApplicationId(ApplicationInstanceReference appRef) { - TenantName tenantName = TenantName.from(appRef.tenantId().toString()); - // Now for the application/instance pair we need to split this - String appNameStr = appRef.applicationInstanceId().toString(); + String appNameStr = appRef.toString(); String[] appNameParts = appNameStr.split(":"); - - // We assume a valid application reference has at lest two parts appname:instancename - // TODO is this assumption valid? - if (appNameParts.length < 2) { - // TODO Since this is used internally we should perhapes use another exception type? - throw new IllegalArgumentException("Application reference not valid: " + appRef); + // TODO model ApplicationInstanceReference properly and validate this there + if (appNameParts.length != 5) { + throw new IllegalArgumentException("Application reference not valid (not 5 parts): " + appRef); } - // Last part of string is the instance name - InstanceName instanceName = InstanceName.from(appNameParts[appNameParts.length-1]); - - // The rest is application - int whereAppNameEnds = appNameStr.lastIndexOf(":"); - ApplicationName appName = ApplicationName.from(appNameStr.substring(0, whereAppNameEnds)); - - return ApplicationId.from(tenantName, appName, instanceName); + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.from(appNameParts[4])); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/VespaModelUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/VespaModelUtil.java index 148c9463f62..e16ccdca5b7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/VespaModelUtil.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/VespaModelUtil.java @@ -25,7 +25,7 @@ import static com.yahoo.collections.CollectionUtil.first; * Utility methods for working with Vespa-specific model entities (see OrchestratorUtil * for more generic model utilities). * - * @author hakon + * @author hakonhall */ public class VespaModelUtil { private static final Logger log = Logger.getLogger(VespaModelUtil.class.getName()); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 79f91ff4255..0e81fdd390c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -8,7 +8,7 @@ import java.io.IOException; /** * Default implementation of the ClusterControllerClient. * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class ClusterControllerClientImpl implements ClusterControllerClient{ public static final String REQUEST_REASON = "Orchestrator"; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerJaxRsApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerJaxRsApi.java index e49173d84bf..d6c4bda8871 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerJaxRsApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerJaxRsApi.java @@ -9,7 +9,7 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; /** - * @author hakon + * @author hakonhall */ public interface ClusterControllerJaxRsApi { @POST diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java index 58f1ef32b10..e5db0ec9795 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerState.java @@ -9,7 +9,7 @@ import com.fasterxml.jackson.annotation.JsonValue; * that class is already fairly complicated, and may perhaps best be screened from JSON annotations - the only * thing we need is the enum < - > String conversions). * - * @author hakon + * @author hakonhall */ public enum ClusterControllerState { MAINTENANCE("maintenance"), diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java index 870ba79158c..a7680d9659d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java @@ -8,7 +8,7 @@ import java.util.Map; import java.util.Objects; /** - * @author hakon + * @author hakonhall */ public class ClusterControllerStateRequest { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateResponse.java index 16d64566b8e..6036cad9aaa 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateResponse.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * The response returned by the cluster controller's set-node-state APIs. * - * @author hakon + * @author hakonhall */ public class ClusterControllerStateResponse { @JsonProperty("wasModified") diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ServiceClusterSuspendPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ServiceClusterSuspendPolicy.java index b2fdd7878d3..7cb20cb8657 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ServiceClusterSuspendPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ServiceClusterSuspendPolicy.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.orchestrator.VespaModelUtil; import com.yahoo.vespa.applicationmodel.ServiceCluster; /** - * @author hakon + * @author hakonhall * @author bakksjo */ public final class ServiceClusterSuspendPolicy { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java index f3d87db5d20..7006b6f6b85 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java @@ -22,7 +22,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ @Path(ApplicationSuspensionApi.PATH_PREFIX) public class ApplicationSuspensionResource implements ApplicationSuspensionApi { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationInstanceStatus.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationInstanceStatus.java index 9c53fb5f5ef..0f5fe530c84 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationInstanceStatus.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationInstanceStatus.java @@ -12,7 +12,7 @@ package com.yahoo.vespa.orchestrator.status; * * @see HostStatus * @author andreer - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public enum ApplicationInstanceStatus { NO_REMARKS, diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java index e1b73c1fe65..d48bc1f302b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java @@ -37,7 +37,7 @@ public class DummyInstanceLookupService implements InstanceLookupService { static { apps.add(new ApplicationInstance<>( new TenantId("test-tenant-id"), - new ApplicationInstanceId("application:instance"), + new ApplicationInstanceId("application:prod:utopia-1:instance"), TestUtil.makeServiceClusterSet( new ServiceCluster<>( new ClusterId("test-cluster-id-1"), @@ -49,7 +49,7 @@ public class DummyInstanceLookupService implements InstanceLookupService { ServiceMonitorStatus.UP), new ServiceInstance<>( new ConfigId("storage/storage/2"), - new HostName("test2.prod.utpoia-1.vespahosted.ut1.yahoo.com"), + new HostName("test2.prod.utopoia-1.vespahosted.ut1.yahoo.com"), ServiceMonitorStatus.UP))), new ServiceCluster<>( new ClusterId("clustercontroller"), @@ -65,7 +65,7 @@ public class DummyInstanceLookupService implements InstanceLookupService { apps.add(new ApplicationInstance<>( new TenantId("mediasearch"), - new ApplicationInstanceId("imagesearch:default"), + new ApplicationInstanceId("imagesearch:prod:utopia-1:default"), TestUtil.makeServiceClusterSet( new ServiceCluster<>( new ClusterId("image"), @@ -93,7 +93,7 @@ public class DummyInstanceLookupService implements InstanceLookupService { apps.add(new ApplicationInstance<>( new TenantId("tenant-id-3"), - new ApplicationInstanceId("application-instance-3:default"), + new ApplicationInstanceId("application-instance-3:prod:utopia-1:default"), TestUtil.makeServiceClusterSet( new ServiceCluster<>( new ClusterId("cluster-id-3"), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 5c1d9bd45be..548d67689d2 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -39,7 +39,7 @@ import static org.mockito.Mockito.spy; /** * Test Orchestrator with a mock backend (the InMemoryStatusService) * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class OrchestratorImplTest { @@ -216,7 +216,7 @@ public class OrchestratorImplTest { InstanceLookupService service = new DummyInstanceLookupService(); String applicationInstanceId = service.findInstanceByHost(DummyInstanceLookupService.TEST1_HOST_NAME).get() .reference().toString(); - assertEquals("test-tenant-id:application:instance", applicationInstanceId); + assertEquals("test-tenant-id:application:prod:utopia-1:instance", applicationInstanceId); } @Test @@ -291,7 +291,7 @@ public class OrchestratorImplTest { private boolean isInMaintenance(ApplicationId appId, HostName hostName) throws ApplicationIdNotFoundException { for (ApplicationInstance<ServiceMonitorStatus> app : DummyInstanceLookupService.getApplications()) { - if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId))) { + if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId,new DummyInstanceLookupService()))) { return clustercontroller.isInMaintenance(app, hostName); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java index 0626bf72e60..2655f6b2b49 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java @@ -12,18 +12,18 @@ import org.junit.Assert; import org.junit.Test; /** - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class OrchestratorUtilTest { private static final ApplicationId APPID_1 = ApplicationId.from( TenantName.from("mediasearch"), - ApplicationName.from("tumblr-search"), + ApplicationName.from("imagesearch"), InstanceName.defaultName()); private static final ApplicationInstanceReference APPREF_1 = new ApplicationInstanceReference( - new TenantId("test-tenant"), - new ApplicationInstanceId("test-application:test-environment:test-region:test-instance-key")); + new TenantId("test-tenant-id"), + new ApplicationInstanceId("application:prod:utopia-1:instance")); /** * Here we don't care how the internal of the different application @@ -35,15 +35,15 @@ public class OrchestratorUtilTest { public void applicationid_conversion_are_symmetric() throws Exception { // From appId to appRef and back - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1); + ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1, new DummyInstanceLookupService()); ApplicationId appIdRoundTrip = OrchestratorUtil.toApplicationId(appRef); Assert.assertEquals(APPID_1, appIdRoundTrip); // From appRef to appId and back ApplicationId appId = OrchestratorUtil.toApplicationId(APPREF_1); - ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId); + ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId, new DummyInstanceLookupService()); Assert.assertEquals(APPREF_1, appRefRoundTrip); } -}
\ No newline at end of file +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java index ab533528cd4..d96b85d50bf 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/VespaModelUtilTest.java @@ -25,7 +25,7 @@ import static junit.framework.TestCase.assertTrue; import static org.fest.assertions.Assertions.assertThat; /** - * @author hakon + * @author hakonhall */ public class VespaModelUtilTest { // Cluster Controller Service Cluster diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java index a682bfe8856..4fc9ad2d56d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -20,7 +20,7 @@ import java.util.Set; * Mock implementation of ClusterControllerClient * <p> * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class ClusterControllerClientFactoryMock implements ClusterControllerClientFactory { Map<String, ClusterControllerState> nodes = new HashMap<>(); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index b8396a51e1e..f3176852bc5 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -26,7 +26,7 @@ import static org.junit.Assert.assertEquals; /** * Tests the implementation of the orchestrators ApplicationAPI. * - * @author <a href="mailto:smorgrav@yahoo-inc.com">Toby</a> + * @author smorgrav */ public class ApplicationSuspensionResourceTest { @@ -140,4 +140,4 @@ public class ApplicationSuspensionResourceTest { assertEquals(1, responses.size()); assertEquals(RESOURCE_2, responses.iterator().next()); } -}
\ No newline at end of file +} diff --git a/parent/.gitignore b/parent/.gitignore deleted file mode 100644 index a5cfcc68fe4..00000000000 --- a/parent/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -pom.xml.build -/pom.xml.build.orig -/target diff --git a/parent/OWNERS b/parent/OWNERS deleted file mode 100644 index 702e6e3eb12..00000000000 --- a/parent/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -gjoranv -ean diff --git a/parent/org.jvnet.hudson/.gitignore b/parent/org.jvnet.hudson/.gitignore deleted file mode 100644 index ea8c4bf7f35..00000000000 --- a/parent/org.jvnet.hudson/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/parent/org.jvnet.hudson/LICENSE b/parent/org.jvnet.hudson/LICENSE deleted file mode 100644 index 2568a8342c2..00000000000 --- a/parent/org.jvnet.hudson/LICENSE +++ /dev/null @@ -1,339 +0,0 @@ -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 - -1. Definitions. - -1.1. Contributor means each individual or entity that creates or -contributes to the creation of Modifications. - -1.2. Contributor Version means the combination of the Original -Software, prior Modifications used by a Contributor (if any), -and the Modifications made by that particular Contributor. - -1.3. Covered Software means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing -Original Software with files containing Modifications, in each -case including portions thereof. - -1.4. Executable means the Covered Software in any form other -than Source Code. - -1.5. Initial Developer means the individual or entity that first -makes Original Software available under this License. - -1.6. Larger Workmeans a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License. - -1.7. License means this document. - -1.8. Licensable means having the right to grant, to the maximum -extent possible, whether at the time of the initial grant or -subsequently acquired, any and all of the rights conveyed herein. - -1.9. Modifications means the Source Code and Executable form of -any of the following: A. Any file that results from an addition -to, deletion from or modification of the contents of a file -containing Original Software or previous Modifications; B. Any -new file that contains any part of the Original Software or -previous Modification; or C. Any new file that is contributed or -otherwise made available under the terms of this License. - -1.10. Original Software means the Source Code and Executable -form of computer software code that is originally released under -this License. - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, -process, and apparatus claims, in any patent Licensable by -grantor. - -1.12. Source Code means (a) the common form of computer software -code in which modifications are made and (b) associated -documentation included in or with such code. - -1.13. You (or Your) means an individual or a legal entity -exercising rights under, and complying with all of the terms of, -this License. For legal entities, You includes any entity which -controls, is controlled by, or is under common control with You. -For purposes of this definition, control means (a) the power, -direct or indirect, to cause the direction or management of such -entity, whether by contract or otherwise, or (b) ownership of -more than fifty percent (50%) of the outstanding shares or -beneficial ownership of such entity. - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your -compliance with Section 3.1 below and subject to third party -intellectual property claims, the Initial Developer hereby -grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Initial Developer, to use, reproduce, -modify, display, perform, sublicense and distribute the Original -Software (or portions thereof), with or without Modifications, -and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using or -selling of Original Software, to make, have made, use, practice, -sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party -under the terms of this License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original -Software, or (2) for infringements caused by: (i) the -modification of the Original Software, or (ii) the combination -of the Original Software with other software or devices. - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual -property claims, each Contributor hereby grants You a -world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, -display, perform, sublicense and distribute the Modifications -created by such Contributor (or portions thereof), either on an -unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using, or -selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions -of such combination), to make, use, sell, offer for sale, have -made, and/or otherwise dispose of: (1) Modifications made by -that Contributor (or portions thereof); and (2) the combination -of Modifications made by that Contributor with its Contributor -Version (or portions of such combination). - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) -areeffective on the date Contributor first distributes or -otherwise makes the Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) above, no patent license is -granted: (1) for any code that Contributor has deleted from the -Contributor Version; (2) for infringements caused by: (i) third -party modifications of Contributor Version, or (ii) the -combination of Modifications made by that Contributor with other -software (except as part of the Contributor Version) or other -devices; or (3) under Patent Claims infringed by Covered -Software in the absence of Modifications made by that -Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. Any Covered Software that You -distribute or otherwise make available in Executable form must -also be made available in Source Code form and that Source Code -form must be distributed only under the terms of this License. -You must include a copy of this License with every copy of the -Source Code form of the Covered Software You distribute or -otherwise make available. You must inform recipients of any such -Covered Software in Executable form as to how they can obtain -such Covered Software in Source Code form in a reasonable manner -on or through a medium customarily used for software exchange. - -3.2. Modifications. The Modifications that You create or to -which You contribute are governed by the terms of this License. -You represent that You believe Your Modifications are Your -original creation(s) and/or You have sufficient rights to grant -the rights conveyed by this License. - -3.3. Required Notices. You must include a notice in each of Your -Modifications that identifies You as the Contributor of the -Modification. You may not remove or alter any copyright, patent -or trademark notices contained within the Covered Software, or -any notices of licensing or any descriptive text giving -attribution to any Contributor or the Initial Developer. - -3.4. Application of Additional Terms. You may not offer or -impose any terms on any Covered Software in Source Code form -that alters or restricts the applicable version of this License -or the recipients rights hereunder. You may choose to offer, and -to charge a fee for, warranty, support, indemnity or liability -obligations to one or more recipients of Covered -Software. However, you may do so only on Your own behalf, and -not on behalf of the Initial Developer or any Contributor. You -must make it absolutely clear that any such warranty, support, -indemnity or liability obligation is offered by You alone, and -You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer -or such Contributor as a result of warranty, support, indemnity -or liability terms You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this -License or under the terms of a license of Your choice, which -may contain terms different from this License, provided that You -are in compliance with the terms of this License and that the -license for the Executable form does not attempt to limit or -alter the recipients rights in the Source Code form from the -rights set forth in this License. If You distribute the Covered -Software in Executable form under a different license, You must -make it absolutely clear that any terms which differ from this -License are offered by You alone, not by the Initial Developer -or Contributor. You hereby agree to indemnify the Initial -Developer and every Contributor for any liability incurred by -the Initial Developer or such Contributor as a result of any -such terms You offer. - -3.6. Larger Works. You may create a Larger Work by combining -Covered Software with other code not governed by the terms of -this License and distribute the Larger Work as a single product. -In such a case, You must make sure the requirements of this -License are fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. Sun Microsystems, Inc. is the initial license -steward and may publish revised and/or new versions of this -License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section -4.3, no one other than the license steward has the right to -modify this License. - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available -under the terms of the version of the License under which You -originally received the Covered Software. If the Initial -Developer includes a notice in the Original Software prohibiting -it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make -the Covered Software available under the terms of the version of -the License under which You originally received the Covered -Software. Otherwise, You may also choose to use, distribute or -otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward. - -4.3. Modified Versions. When You are an Initial Developer and -You want to create a new license for Your Original Software, You -may create and use a modified version of this License if You: -(a) rename the license and remove any references to the name of -the license steward (except to note that the license differs -from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License. - -5. DISCLAIMER OF WARRANTY. COVERED SOFTWARE IS PROVIDED UNDER -THIS LICENSE ON AN AS IS BASIS, WITHOUT WARRANTY OF ANY KIND, -EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, -WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. -THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE -DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY -OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, -REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN -ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE -IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein -and fail to cure such breach within 30 days of becoming aware of -the breach. Provisions which, by their nature, must remain in -effect beyond the termination of this License shall survive. - -6.2. If You assert a patent infringement claim (excluding -declaratory judgment actions) against Initial Developer or a -Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging -that the Participant Software (meaning the Contributor Version -where the Participant is a Contributor or the Original Software -where the Participant is the Initial Developer) directly or -indirectly infringes any patent, then any and all rights granted -directly or indirectly to You by such Participant, the Initial -Developer (if the Initial Developer is not the Participant) and -all Contributors under Sections 2.1 and/or 2.2 of this License -shall, upon 60 days notice from Participant terminate -prospectively and automatically at the expiration of such 60 day -notice period, unless if within such 60 day period You withdraw -Your claim with respect to the Participant Software against such -Participant either unilaterally or pursuant to a written -agreement with Participant. - -6.3. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by -You or any distributor hereunder prior to termination (excluding -licenses granted to You by any distributor) shall survive -termination. - -7. LIMITATION OF LIABILITY. UNDER NO CIRCUMSTANCES AND UNDER NO -LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR -OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER -CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY -INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST -PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR -MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, -EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY -OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH -PARTYS NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH -LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS -EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. The Covered Software is a -commercial item, as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of commercial computer software (as that -term is defined at 48 C.F.R. 252.227-7014(a)(1)) and commercial -computer software documentation as such terms are used in 48 -C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and -48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all -U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights -clause is in lieu of, and supersedes, any other FAR, DFAR, or -other clause or provision that addresses Government rights in -computer software under this License. - -9. MISCELLANEOUS. This License represents the complete agreement -concerning subject matter hereof. If any provision of this -License is held to be unenforceable, such provision shall be -reformed only to the extent necessary to make it enforceable. -This License shall be governed by the law of the jurisdiction -specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides -otherwise), excluding such jurisdictions conflict-of-law -provisions. Any litigation relating to this License shall be -subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within -the Original Software, with the losing party responsible for -costs, including, without limitation, court costs and reasonable -attorneys fees and expenses. The application of the United -Nations Convention on Contracts for the International Sale of -Goods is expressly excluded. Any law or regulation which -provides that the language of a contract shall be construed -against the drafter shall not apply to this License. You agree -that You alone are responsible for compliance with the United -States export administration regulations (and the export control -laws and regulation of any other countries) when You use, -distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. As between Initial Developer and -the Contributors, each party is responsible for claims and -damages arising, directly or indirectly, out of its utilization -of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on -an equitable basis. Nothing herein is intended or shall be -deemed to constitute any admission of liability. - ----------------------------------------------------------------- - -NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND -DISTRIBUTION LICENSE (CDDL): This code is released under the -CDDL and shall be governed by the laws of the State of -California (excluding conflict-of-law provisions). Any -litigation relating to this License shall be subject to the -jurisdiction of the Federal Courts of the Northern District of -California and the state courts of the State of California, with -venue lying in Santa Clara County, California. - ----------------------------------------------------------------- diff --git a/parent/org.jvnet.hudson/pom.xml b/parent/org.jvnet.hudson/pom.xml deleted file mode 100644 index bebb4a26ac2..00000000000 --- a/parent/org.jvnet.hudson/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <groupId>org.jvnet.hudson</groupId> - <artifactId>annotation-indexer</artifactId> - <name>annotation-indexer</name> - <version>1.2</version> - <description> - Creates index of annotations. - </description> - - <dependencies> - <dependency> - <groupId>org.kohsuke.metainf-services</groupId> - <artifactId>metainf-services</artifactId> - <version>1.1</version> - <optional>true</optional> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>3.8.1</version> - <scope>test</scope> - </dependency> - </dependencies> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - -</project> diff --git a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/AnnotationProcessorImpl.java b/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/AnnotationProcessorImpl.java deleted file mode 100644 index 3a7b6c2de62..00000000000 --- a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/AnnotationProcessorImpl.java +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * The contents of this file are subject to the terms of the Common Development - * and Distribution License (the License). You may not use this file except in - * compliance with the License. - * - * You can obtain a copy of the License at http://www.sun.com/cddl/cddl.html - * or http://www.netbeans.org/cddl.txt. - * - * When distributing Covered Code, include this CDDL Header Notice in each file - * and include the License file at http://www.netbeans.org/cddl.txt. - * If applicable, add the following below the CDDL Header, with the fields - * enclosed by brackets [] replaced by your own identifying information: - * "Portions Copyrighted [year] [name of copyright owner]" - * - * The Original Software is SezPoz. The Initial Developer of the Original - * Software is Sun Microsystems, Inc. Portions Copyright 2008 Sun - * Microsystems, Inc. All Rights Reserved. - */ -package org.jvnet.hudson.annotation_indexer; - -import org.kohsuke.MetaInfServices; - -import javax.annotation.processing.AbstractProcessor; -import javax.annotation.processing.Processor; -import javax.annotation.processing.RoundEnvironment; -import javax.annotation.processing.SupportedAnnotationTypes; -import javax.annotation.processing.SupportedSourceVersion; -import static javax.lang.model.SourceVersion.RELEASE_6; -import static javax.lang.model.SourceVersion.RELEASE_7; -import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.Element; -import javax.lang.model.element.TypeElement; -import javax.lang.model.util.Elements; -import javax.tools.Diagnostic.Kind; -import javax.tools.FileObject; -import static javax.tools.StandardLocation.CLASS_OUTPUT; -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -/** - * Creates indices of {@link Indexed} annotations. - * - * @author Kohsuke Kawaguchi - */ -@SupportedSourceVersion(RELEASE_7) -@SupportedAnnotationTypes("*") -@SuppressWarnings({"Since15"}) -@MetaInfServices(Processor.class) -public class AnnotationProcessorImpl extends AbstractProcessor { - /** - * Use of an annotation. - */ - private final class Use { - /** - * FQCN of the annotation. - */ - final String annotationName; - /** - * Strings that designate FQCNs where annotations are used, either on a class or its members. - */ - final Set<String> classes = new TreeSet<String>(); - /** - * Keeps track of elements that has the annotation. - */ - final Set<Element> originatingElements = new HashSet<Element>(); - - private Use(String annotationName) { - this.annotationName = annotationName; - } - - void add(Element elt) { - originatingElements.add(elt); - - TypeElement t; - switch (elt.getKind()) { - case CLASS: - t = (TypeElement) elt; - break; - case METHOD: - case FIELD: - t = (TypeElement) elt.getEnclosingElement(); - break; - default: - throw new AssertionError(elt.getKind()); - } - classes.add(getElementUtils().getBinaryName(t).toString()); - } - - String getIndexFileName() { - return "META-INF/annotations/" + annotationName; - } - - /** - * Loads existing index, if it exists. - */ - List<String> loadExisting() throws IOException { - List<String> elements = new ArrayList<String>(); - try { - FileObject in = processingEnv.getFiler().getResource(CLASS_OUTPUT, "", getIndexFileName()); - // Read existing annotations, for incremental compilation. - BufferedReader is = new BufferedReader(new InputStreamReader(in.openInputStream(),"UTF-8")); - try { - String line; - while ((line=is.readLine())!=null) - elements.add(line); - } finally { - is.close(); - } - } catch (FileNotFoundException x) { - // OK, created for the first time - } - return elements; - } - - void write() { - try { - FileObject out = processingEnv.getFiler().createResource(CLASS_OUTPUT, - "", getIndexFileName(), - originatingElements.toArray(new Element[originatingElements.size()])); - - PrintWriter w = new PrintWriter(new OutputStreamWriter(out.openOutputStream(),"UTF-8")); - try { - for (String el : classes) - w.println(el); - } finally { - w.close(); - } - } catch (IOException x) { - processingEnv.getMessager().printMessage(Kind.ERROR, x.toString()); - } - } - } - - private Elements getElementUtils() { - return processingEnv.getElementUtils(); - } - - @Override - public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) { - if (roundEnv.processingOver()) - return false; - - // map from indexable annotation names, to actual uses - Map<String,Use> output = new HashMap<String,Use>(); - scan(annotations, roundEnv, output); - for (Use u : output.values()) - u.write(); - return false; - } - - private AnnotationMirror findAnnotationOn(Element e, String name) { - for (AnnotationMirror a : getElementUtils().getAllAnnotationMirrors(e)) - if (getElementUtils().getBinaryName((TypeElement) a.getAnnotationType().asElement()).contentEquals(name)) - return a; - return null; - } - - private void scan(Set<? extends TypeElement> annotations, - RoundEnvironment roundEnv, Map<String,Use> output) { - for (TypeElement ann : annotations) { - AnnotationMirror indexed = findAnnotationOn(ann,Indexed.class.getName()); - if (indexed == null) - continue; // not indexed - - String annName = getElementUtils().getBinaryName(ann).toString(); - Use o = output.get(annName); - if (o==null) - output.put(annName,o=new Use(annName)); - - for (Element elt : roundEnv.getElementsAnnotatedWith(ann)) { - AnnotationMirror marked = findAnnotationOn(elt,annName); - assert marked != null; - - // TODO: validator support - - o.add(elt); - } - } - } - - -} diff --git a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/FilterIterator.java b/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/FilterIterator.java deleted file mode 100644 index 6e0035ddeb0..00000000000 --- a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/FilterIterator.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * The contents of this file are subject to the terms of the Common Development - * and Distribution License (the License). You may not use this file except in - * compliance with the License. - * - * You can obtain a copy of the License at http://www.sun.com/cddl/cddl.html - * or http://www.netbeans.org/cddl.txt. - * - * When distributing Covered Code, include this CDDL Header Notice in each file - * and include the License file at http://www.netbeans.org/cddl.txt. - * If applicable, add the following below the CDDL Header, with the fields - * enclosed by brackets [] replaced by your own identifying information: - * "Portions Copyrighted [year] [name of copyright owner]" - * - * The Original Software is SezPoz. The Initial Developer of the Original - * Software is Sun Microsystems, Inc. Portions Copyright 2008 Sun - * Microsystems, Inc. All Rights Reserved. - */package org.jvnet.hudson.annotation_indexer; - -import java.util.Iterator; -import java.util.NoSuchElementException; - -/** - * @author Kohsuke Kawaguchi - */ -abstract class FilterIterator<T> implements Iterator<T> { - private final Iterator<? extends T> core; - private T next; - private boolean fetched; - - protected FilterIterator(Iterator<? extends T> core) { - this.core = core; - } - - protected FilterIterator(Iterable<? extends T> core) { - this(core.iterator()); - } - - private void fetch() { - while(!fetched && core.hasNext()) { - T n = core.next(); - if(filter(n)) { - next = n; - fetched = true; - } - } - } - - /** - * Filter out items in the original collection. - * - * @return - * true to leave this item and return this item from this iterator. - * false to hide this item. - */ - protected abstract boolean filter(T t); - - public boolean hasNext() { - fetch(); - return fetched; - } - - public T next() { - fetch(); - if(!fetched) throw new NoSuchElementException(); - fetched = false; - return next; - } - - public void remove() { - core.remove(); - } -} diff --git a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Index.java b/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Index.java deleted file mode 100644 index b345f6b797d..00000000000 --- a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Index.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * The contents of this file are subject to the terms of the Common Development - * and Distribution License (the License). You may not use this file except in - * compliance with the License. - * - * You can obtain a copy of the License at http://www.sun.com/cddl/cddl.html - * or http://www.netbeans.org/cddl.txt. - * - * When distributing Covered Code, include this CDDL Header Notice in each file - * and include the License file at http://www.netbeans.org/cddl.txt. - * If applicable, add the following below the CDDL Header, with the fields - * enclosed by brackets [] replaced by your own identifying information: - * "Portions Copyrighted [year] [name of copyright owner]" - * - * The Original Software is SezPoz. The Initial Developer of the Original - * Software is Sun Microsystems, Inc. Portions Copyright 2008 Sun - * Microsystems, Inc. All Rights Reserved. - */package org.jvnet.hudson.annotation_indexer; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.lang.annotation.Annotation; -import java.lang.reflect.AnnotatedElement; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.net.URL; -import java.util.Enumeration; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author Kohsuke Kawaguchi - */ -public class Index { - /** - * Lists up all the elements annotated by the given annotation and of the given {@link AnnotatedElement} subtype. - */ - public static <T extends AnnotatedElement> Iterable<T> list(Class<? extends Annotation> type, ClassLoader cl, final Class<T> subType) throws IOException { - final Iterable<AnnotatedElement> base = list(type,cl); - return new Iterable<T>() { - public Iterator<T> iterator() { - return new FilterIterator(base.iterator()) { - protected boolean filter(Object o) { - return subType.isInstance(o); - } - }; - } - }; - } - - /** - * Lists up all the elements annotated by the given annotation. - */ - public static Iterable<AnnotatedElement> list(final Class<? extends Annotation> type, final ClassLoader cl) throws IOException { - if (!type.isAnnotationPresent(Indexed.class)) - throw new IllegalArgumentException(type+" doesn't have @Indexed"); - - final Set<String> ids = new TreeSet<String>(); - - final Enumeration<URL> res = cl.getResources("META-INF/annotations/"+type.getName()); - while (res.hasMoreElements()) { - URL url = res.nextElement(); - BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream(), "UTF-8")); - String line; - while ((line=r.readLine())!=null) - ids.add(line); - } - - return new Iterable<AnnotatedElement>() { - public Iterator<AnnotatedElement> iterator() { - return new Iterator<AnnotatedElement>() { - /** - * Next element to return. - */ - private AnnotatedElement next; - - private Iterator<String> iditr = ids.iterator(); - - private List<AnnotatedElement> lookaheads = new LinkedList<AnnotatedElement>(); - - public boolean hasNext() { - fetch(); - return next!=null; - } - - public AnnotatedElement next() { - fetch(); - AnnotatedElement r = next; - next = null; - return r; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - private void fetch() { - while (next==null) { - if (!lookaheads.isEmpty()) { - next = lookaheads.remove(0); - return; - } - - if (!iditr.hasNext()) return; - String name = iditr.next(); - - try { - Class<?> c = cl.loadClass(name); - - if (c.isAnnotationPresent(type)) - lookaheads.add(c); - listAnnotatedElements(c.getDeclaredMethods()); - listAnnotatedElements(c.getDeclaredFields()); - } catch (ClassNotFoundException e) { - LOGGER.log(Level.FINE, "Failed to load: "+name,e); - } - } - } - - private void listAnnotatedElements(AnnotatedElement[] elements) { - for (AnnotatedElement m : elements) { - // this means we don't correctly handle - if (m.isAnnotationPresent(type)) - lookaheads.add(m); - } - } - }; - } - }; - } - - private static final Logger LOGGER = Logger.getLogger(Index.class.getName()); -} diff --git a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Indexed.java b/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Indexed.java deleted file mode 100644 index 25be039f44c..00000000000 --- a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Indexed.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * The contents of this file are subject to the terms of the Common Development - * and Distribution License (the License). You may not use this file except in - * compliance with the License. - * - * You can obtain a copy of the License at http://www.sun.com/cddl/cddl.html - * or http://www.netbeans.org/cddl.txt. - * - * When distributing Covered Code, include this CDDL Header Notice in each file - * and include the License file at http://www.netbeans.org/cddl.txt. - * If applicable, add the following below the CDDL Header, with the fields - * enclosed by brackets [] replaced by your own identifying information: - * "Portions Copyrighted [year] [name of copyright owner]" - * - * The Original Software is SezPoz. The Initial Developer of the Original - * Software is Sun Microsystems, Inc. Portions Copyright 2008 Sun - * Microsystems, Inc. All Rights Reserved. - */ -package org.jvnet.hudson.annotation_indexer; - -import java.lang.annotation.Documented; -import static java.lang.annotation.ElementType.ANNOTATION_TYPE; -import java.lang.annotation.Retention; -import static java.lang.annotation.RetentionPolicy.RUNTIME; -import java.lang.annotation.Target; - -/** - * Marks annotations that should be indexed during compilation. - * - * @author Kohsuke Kawaguchi - */ -@Documented -@Retention(RUNTIME) -@Target(ANNOTATION_TYPE) -public @interface Indexed { - Class<? extends Validator>[] validators() default {}; -} diff --git a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Validator.java b/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Validator.java deleted file mode 100644 index 4015d2588f8..00000000000 --- a/parent/org.jvnet.hudson/src/main/java/org/jvnet/hudson/annotation_indexer/Validator.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * The contents of this file are subject to the terms of the Common Development - * and Distribution License (the License). You may not use this file except in - * compliance with the License. - * - * You can obtain a copy of the License at http://www.sun.com/cddl/cddl.html - * or http://www.netbeans.org/cddl.txt. - * - * When distributing Covered Code, include this CDDL Header Notice in each file - * and include the License file at http://www.netbeans.org/cddl.txt. - * If applicable, add the following below the CDDL Header, with the fields - * enclosed by brackets [] replaced by your own identifying information: - * "Portions Copyrighted [year] [name of copyright owner]" - * - * The Original Software is SezPoz. The Initial Developer of the Original - * Software is Sun Microsystems, Inc. Portions Copyright 2008 Sun - * Microsystems, Inc. All Rights Reserved. - */ -package org.jvnet.hudson.annotation_indexer; - -import javax.annotation.processing.RoundEnvironment; -import javax.annotation.processing.ProcessingEnvironment; -import javax.lang.model.element.Element; - -/** - * Checkes the usage of {@link Indexed} annotations at compile-time. - * - * @author Kohsuke Kawaguchi - * @see Indexed - */ -public interface Validator { - /** - * Checks the occurrence of the {@link Indexed} annotation - * and report any error. Useful for early error detection. - */ - void check(Element use, RoundEnvironment e, ProcessingEnvironment env); -} diff --git a/parent/pom.xml b/parent/pom.xml deleted file mode 100644 index 11e1c96cd90..00000000000 --- a/parent/pom.xml +++ /dev/null @@ -1,1114 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <packaging>pom</packaging> - <version>6-SNAPSHOT</version> - <name>parent</name> - <description>Parent artifact for all Vespa maven projects.</description> - <pluginRepositories> - <pluginRepository> - <id>scala-tools.org</id> - <name>Scala-tools Maven2 Repository</name> - <url>http://scala-tools.org/repo-releases</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </pluginRepository> - </pluginRepositories> - <repositories> - <repository> - <id>scala-tools.org</id> - <name>Scala-tools Maven2 Repository</name> - <url>http://scala-tools.org/repo-releases</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> - <build> - <finalName>${project.artifactId}</finalName> - <extensions> - <extension> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-ssh-external</artifactId> - <version>2.7</version> - </extension> - <extension> - <groupId>org.apache.maven.archetype</groupId> - <artifactId>archetype-packaging</artifactId> - <version>2.0</version> - </extension> - </extensions> - <pluginManagement> - <plugins> - <plugin> - <groupId>com.github.goldin</groupId> - <artifactId>copy-maven-plugin</artifactId> - <version>0.2.5</version> - </plugin> - <plugin> - <groupId>com.infradna.tool</groupId> - <artifactId>bridge-method-injector</artifactId> - <version>1.4</version> - </plugin> - <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr3-maven-plugin</artifactId> - <version>3.5.2</version> - </plugin> - <plugin> - <groupId>org.apache.felix</groupId> - <artifactId>maven-bundle-plugin</artifactId> - <version>2.4.0</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-archetype-plugin</artifactId> - <version>2.0</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <version>2.4</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <source>1.8</source> - <target>1.8</target> - <showWarnings>true</showWarnings> - <optimize>true</optimize> - <showDeprecation>false</showDeprecation> - <compilerArgs> - <arg>-Xlint:all</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>2.8</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <version>2.5</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <configuration> - <additionalparam>-Xdoclint:${doclint} -Xdoclint:-missing</additionalparam> - </configuration> - <version>2.9</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-plugin-plugin</artifactId> - <version>3.3</version> - <configuration> - <!-- see http://jira.codehaus.org/browse/MNG-5346 --> - <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> - </configuration> - <executions> - <execution> - <id>mojo-descriptor</id> - <goals> - <goal>descriptor</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-project-info-reports-plugin</artifactId> - <version>2.7</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> - <version>2.5</version> - <configuration> - <escapeString>\</escapeString> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> - <version>3.3</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>2.1.2</version> - <configuration> - <includePom>true</includePom> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.16</version> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <systemPropertyVariables> - <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> - </systemPropertyVariables> - <excludes> - <exclude>**/*SystemTest.java</exclude> - </excludes> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>2.16</version> - <configuration> - <alwaysGenerateSurefireReport>false</alwaysGenerateSurefireReport> - <showSuccess>false</showSuccess> - </configuration> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.9.1</version> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.2.1</version> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>javacc-maven-plugin</artifactId> - <version>2.6</version> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>properties-maven-plugin</artifactId> - <version>1.0-alpha-2</version> - </plugin> - <plugin> - <groupId>org.scala-tools</groupId> - <artifactId>maven-scala-plugin</artifactId> - <version>2.15.2</version> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${project.version}</version> - <configuration> - <configGenVersion>${project.version}</configGenVersion> - <useCommonAssemblyIds>true</useCommonAssemblyIds> - </configuration> - </plugin> - - <!-- Eclipse-specific stuff, wrong place, but only option, see https://bugs.eclipse.org/bugs/show_bug.cgi?id=350414 --> - <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.--> - <plugin> - <groupId>org.eclipse.m2e</groupId> - <artifactId>lifecycle-mapping</artifactId> - <version>1.0.0</version> - <configuration> - <lifecycleMappingMetadata> - <pluginExecutions> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.scala-tools</groupId> - <artifactId>maven-scala-plugin</artifactId> - <versionRange>[2.15.2,)</versionRange> - <goals> - <goal>add-source</goal> - <goal>testCompile</goal> - <goal>compile</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.codehaus.mojo</groupId> - <artifactId>properties-maven-plugin</artifactId> - <versionRange>[1.0-alpha-2,)</versionRange> - <goals> - <goal>write-project-properties</goal> - <goal>read-project-properties</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <versionRange>[1.1,)</versionRange> - <goals> - <goal>java</goal> - <goal>exec</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <versionRange>[1.3,)</versionRange> - <goals> - <goal>run</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.codehaus.mojo</groupId> - <artifactId>javacc-maven-plugin</artifactId> - <versionRange>[2.4,)</versionRange> - <goals> - <goal>javacc</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId> - org.apache.maven.plugins - </groupId> - <artifactId> - maven-jar-plugin - </artifactId> - <versionRange> - [0,) - </versionRange> - <goals> - <goal>jar</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId> - com.yahoo.vespa - </groupId> - <artifactId> - container-maven-plugin - </artifactId> - <versionRange> - [0,) - </versionRange> - <goals> - <goal>generateSources</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.antlr</groupId> - <artifactId> - antlr3-maven-plugin - </artifactId> - <versionRange>[0,)</versionRange> - <goals> - <goal>antlr</goal> - </goals> - </pluginExecutionFilter> - <action> - <execute> - <runOnIncremental>false</runOnIncremental> - <runOnConfiguration>true</runOnConfiguration> - </execute> - </action> - </pluginExecution> - </pluginExecutions> - </lifecycleMappingMetadata> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - <profiles> - <profile> - <id>attach-sources</id> - <activation> - <property> - <name>!skipSources</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <executions> - <execution> - <id>attach-sources</id> - <goals> - <goal>jar-no-fork</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>generate-javadoc</id> - <activation> - <property> - <name>!skipJavadoc</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <executions> - <execution> - <id>generate-javadoc</id> - <phase>package</phase> - <goals> - <goal>javadoc</goal> - </goals> - </execution> - </executions> - <configuration> - <additionalparam>-Xdoclint:${doclint} -Xdoclint:-missing</additionalparam> - <failOnError>${javadoc.failOnError}</failOnError> - <quiet>true</quiet> - <show>private</show> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>coverage</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <configuration> - <includePluginDependencies>true</includePluginDependencies> - </configuration> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>src/main/scala</source> - </sources> - </configuration> - </execution> - <execution> - <id>add-test-source</id> - <phase>generate-test-sources</phase> - <goals> - <goal>add-test-source</goal> - </goals> - <configuration> - <sources> - <source>src/test/scala</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>systemtests</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <excludes> - <exclude>none</exclude> - </excludes> - <includes> - <include>**/*SystemTest.java</include> - </includes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - </profiles> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-ssh-external</artifactId> - <version>2.7</version> - </dependency> - <dependency> - <groupId>com.github.cverges.expect4j</groupId> - <artifactId>expect4j</artifactId> - <version>1.6</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <version>1.8</version> - </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.7</version> - </dependency> - <dependency> - <groupId>aopalliance</groupId> - <artifactId>aopalliance</artifactId> - <version>1.0</version> - </dependency> - <dependency> - <groupId>asm</groupId> - <artifactId>asm</artifactId> - <version>3.3.1</version> - </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>5.0.3</version> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>annotations</artifactId> - <version>1.3.9</version> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - <version>1.3.9</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>18.0</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava-testlib</artifactId> - <version>18.0</version> - </dependency> - <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <version>3.0</version> - </dependency> - <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <version>3.0</version> - <classifier>no_aop</classifier> - </dependency> - <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-assistedinject</artifactId> - <version>3.0</version> - </dependency> - <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-multibindings</artifactId> - <version>3.0</version> - </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>2.4.1</version> - </dependency> - <dependency> - <groupId>com.googlecode.jmockit</groupId> - <artifactId>jmockit</artifactId> - <version>1.2</version> - </dependency> - <dependency> - <groupId>com.goldmansachs</groupId> - <artifactId>gs-collections</artifactId> - <version>6.1.0</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-jaxb-annotations</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-base</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-xml-provider</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jdk8</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.infradna.tool</groupId> - <artifactId>bridge-method-annotation</artifactId> - <version>1.4</version> - </dependency> - <dependency> - <groupId>com.ning</groupId> - <artifactId>async-http-client</artifactId> - <version>1.7.17</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>com.sun.jersey.contribs</groupId> - <artifactId>jersey-guice</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>com.sun.jersey.contribs</groupId> - <artifactId>jersey-multipart</artifactId> - <version>1.13</version> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <version>1.3.1</version> - </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.4</version> - </dependency> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - <version>3.2.1</version> - </dependency> - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - <version>1.6</version> - </dependency> - <dependency> - <groupId>commons-daemon</groupId> - <artifactId>commons-daemon</artifactId> - <version>1.0.3</version> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>2.4</version> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.6</version> - </dependency> - <dependency> - <groupId>commons-net</groupId> - <artifactId>commons-net</artifactId> - <version>2.0</version> - </dependency> - <dependency> - <groupId>commons-pool</groupId> - <artifactId>commons-pool</artifactId> - <version>1.5.6</version> - </dependency> - <!-- Explicitly included to get Zookeeper version 3.4.8, - can be excluded if you want the Zookeeper version - used by curator by default - --> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.4.8</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <!-- WARNING: If you change this version, you also need to update - zkfacade/src/main/java/org/apache/curator/**/package-info.java - using something like - find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ - xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 2, minor = 9, micro = 1/g' - --> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - <version>3.6.5.Final</version> - </dependency> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <version>3.1.0</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.11</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit-dep</artifactId> - <version>4.11</version> - </dependency> - <dependency> - <groupId>org.antlr</groupId> - <artifactId>antlr-runtime</artifactId> - <version>3.5.2</version> - </dependency> - <dependency> - <groupId>org.apache.aries.spifly</groupId> - <artifactId>org.apache.aries.spifly.dynamic.bundle</artifactId> - <version>${aries.spifly.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>3.1</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.framework</artifactId> - <version>4.2.1</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.log</artifactId> - <version>1.0.1</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.main</artifactId> - <version>4.2.1</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>fluent-hc</artifactId> - <version>4.3.6</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.3.6</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>4.3.3</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpmime</artifactId> - <version>4.3.6</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-artifact</artifactId> - <version>3.1.1</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-core</artifactId> - <version>3.1.1</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-model</artifactId> - <version>3.1.1</version> - </dependency> - <dependency> - <groupId>org.apache.maven.plugin-tools</groupId> - <artifactId>maven-plugin-annotations</artifactId> - <version>3.2</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-plugin-api</artifactId> - <version>3.1.1</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> - <artifactId>maven-project</artifactId> - <version>2.2.1</version> - </dependency> - <dependency> - <groupId>org.codehaus.plexus</groupId> - <artifactId>plexus-interactivity-api</artifactId> - <version>1.0-alpha-5</version> - </dependency> - <dependency> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>2.2</version> - </dependency> - <dependency> - <groupId>org.apache.maven.surefire</groupId> - <artifactId>surefire-junit4</artifactId> - <version>2.16</version> - </dependency> - <dependency> - <groupId>org.apache.maven.surefire</groupId> - <artifactId>surefire-providers</artifactId> - <version>2.16</version> - <type>pom</type> - </dependency> - <dependency> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - <version>1.3.1</version> - </dependency> - <dependency> - <groupId>org.cthul</groupId> - <artifactId>cthul-matchers</artifactId> - <version>1.0</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-continuation</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlets</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>websocket-server</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>websocket-servlet</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.grizzly</groupId> - <artifactId>grizzly-websockets</artifactId> - <version>2.3.2</version> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> - <version>1.3</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <version>1.3</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <version>1.3</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>uk.co.datumedge</groupId> - <artifactId>hamcrest-json</artifactId> - <version>0.2</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>20090211</version> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>1.9.5</version> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <version>1.9.5</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.osgi</groupId> - <artifactId>org.osgi.compendium</artifactId> - <version>4.3.0</version> - </dependency> - <dependency> - <groupId>org.osgi</groupId> - <artifactId>org.osgi.core</artifactId> - <version>4.3.0</version> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-compiler</artifactId> - <version>${scala.version}</version> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - <version>${scala.version}</version> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-parser-combinators_${scala.major-version}</artifactId> - <version>1.0.1</version> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <version>1.0.2</version> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - <version>2.2.2</version> - </dependency> - <dependency> - <groupId>org.json4s</groupId> - <artifactId>json4s-native_${scala.major-version}</artifactId> - <version>3.3.0</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-test</artifactId> - <version>4.0.6.RELEASE</version> - </dependency> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>6.9.10</version> - </dependency> - <dependency> - <groupId>org.twdata.maven</groupId> - <artifactId>mojo-executor</artifactId> - <version>2.2.0</version> - </dependency> - <dependency> - <groupId>net.jcip</groupId> - <artifactId>jcip-annotations</artifactId> - <version>1.0</version> - </dependency> - <dependency> - <groupId>net.jpountz.lz4</groupId> - <artifactId>lz4</artifactId> - <version>1.3.0</version> - </dependency> - <dependency> - <groupId>net.spy</groupId> - <artifactId>spymemcached</artifactId> - <version>2.10.1</version> - </dependency> - <dependency> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - <version>2.11.0</version> - </dependency> - <!-- jersey 2 support --> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <version>${javax.ws.rs-api.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.containers</groupId> - <artifactId>jersey-container-servlet-core</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.containers</groupId> - <artifactId>jersey-container-servlet</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-multipart</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-proxy-client</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>com.ibm.icu</groupId> - <artifactId>icu4j</artifactId> - <version>57.1</version> - </dependency> - </dependencies> - </dependencyManagement> - <properties> - <javax.ws.rs-api.version>2.0</javax.ws.rs-api.version> - <aries.spifly.version>1.0.8</aries.spifly.version> - <aries.util.version>1.0.0</aries.util.version> - <asm-debug-all.version>5.0.3</asm-debug-all.version> - <curator.version>2.9.1</curator.version> - <jackson2.version>2.5.3</jackson2.version> - <jersey2.version>2.10.1</jersey2.version> - <jetty.version>9.3.8.v20160314</jetty.version> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <test.hide>true</test.hide> - <doclint>all</doclint> - <scala.major-version>2.11</scala.major-version> - <scala.version>${scala.major-version}.4</scala.version> - </properties> -</project> diff --git a/persistence/.gitignore b/persistence/.gitignore index be0452bed21..4b160dd6a2e 100644 --- a/persistence/.gitignore +++ b/persistence/.gitignore @@ -1,4 +1,3 @@ /target /pom.xml.build Makefile -Testing diff --git a/persistence/OWNERS b/persistence/OWNERS index 97c35339850..11a58a546b1 100644 --- a/persistence/OWNERS +++ b/persistence/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/persistence/pom.xml b/persistence/pom.xml index 3d887b6bf31..57c6093d6b9 100644 --- a/persistence/pom.xml +++ b/persistence/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>persistence</artifactId> <packaging>container-plugin</packaging> diff --git a/persistencetypes/.gitignore b/persistencetypes/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/persistencetypes/.gitignore +++ b/persistencetypes/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/persistencetypes/OWNERS b/persistencetypes/OWNERS index 97c35339850..11a58a546b1 100644 --- a/persistencetypes/OWNERS +++ b/persistencetypes/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis @@ -1,116 +1,1229 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-java-modules</artifactId> - <version>6-SNAPSHOT</version> - <packaging>pom</packaging> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <packaging>pom</packaging> + <version>6-SNAPSHOT</version> + <name>parent</name> + <description>Parent artifact for all Vespa maven projects.</description> + <url>http://yahoo.github.io/vespa</url> - <modules> - <module>annotations</module> - <module>application</module> - <module>application-deploy-plugin</module> - <module>application-model</module> - <module>application-preprocessor</module> - <module>bundle-plugin</module> - <module>chain</module> - <module>clustercontroller-apps</module> - <module>clustercontroller-apputil</module> - <module>clustercontroller-core</module> - <module>clustercontroller-standalone</module> - <module>clustercontroller-utils</module> - <module>component</module> - <module>config-application-package</module> - <module>configdefinitions</module> - <module>configgen</module> - <module>config-bundle</module> - <module>config-class-plugin</module> - <module>config-lib</module> - <module>config-model-api</module> - <module>config-model-fat</module> - <module>config-model</module> - <module>config</module> - <module>config-provisioning</module> - <module>config-proxy</module> - <module>configserver</module> - <module>config_test</module> - <module>container-core</module> - <module>container-accesslogging</module> - <module>container-dev</module> - <module>container-di</module> - <module>container-disc</module> - <module>container-jersey2</module> - <module>container-messagebus</module> - <module>container-search-and-docproc</module> - <module>container-search</module> - <module>container-test-jars</module> - <module>defaults</module> - <module>docproc</module> - <module>docprocs</module> - <module>documentapi</module> - <module>document</module> - <module>documentgen-test</module> - <module>dummy-persistence</module> - <module>fileacquirer</module> - <module>filedistributionmanager</module> - <module>filedistribution</module> - <module>filedistribution_test</module> - <module>fsa</module> - <module>indexinglanguage</module> - <module>jaxrs_client_utils</module> - <module>jaxrs_utils</module> - <module>jdisc_container_maven_archetype_application</module> - <module>jdisc_core</module> - <module>jdisc_core_test</module> - <module>jdisc_http_service</module> - <module>jdisc_jetty</module> - <module>jdisc_jmx_metrics</module> - <module>jdisc_maven_archetype_component</module> - <module>jdisc_messagebus_service</module> - <module>jrt</module> - <module>libmlr</module> - <module>lifecycle_mapping</module> - <module>linguistics</module> - <module>logd</module> - <module>logserver</module> - <module>messagebus-disc</module> - <module>messagebus</module> - <module>metrics</module> - <module>node-repository</module> - <module>node-admin</module> - <module>orchestrator-restapi</module> - <module>orchestrator</module> - <module>parent</module> - <module>persistence</module> - <module>predicate-search</module> - <module>predicate-search-core</module> - <module>processing</module> - <module>provided-dependencies</module> - <module>searchcore</module> - <module>searchlib</module> - <module>searchsummary</module> - <module>serviceview</module> - <module>service-monitor</module> - <module>simplemetrics</module> - <module>socket_test</module> - <module>standalone-container</module> - <module>statistics</module> - <module>storage</module> - <module>testutil</module> - <module>vdslib</module> - <module>vespaclient-core</module> - <module>vespaclient-container-plugin</module> - <module>vespa-application-maven-plugin</module> - <module>vespa-documentgen-plugin</module> - <module>vespa_feed_perf</module> - <module>vespa-http-client</module> - <module>vespa_jersey2</module> - <module>vespajlib</module> - <module>vespalog</module> - <module>vsm</module> - <module>yolean</module> - <module>zkfacade</module> - <module>scalalib</module> - </modules> + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + + <developers> + <developer> + <name>Yahoo Inc.</name> + <url>https://github.com/yahoo</url> + </developer> + </developers> + + <distributionManagement> + <repository> + <id>bintray-vespa-repo</id> + <url>https://api.bintray.com/maven/yahoo/maven/vespa;publish=1</url> + </repository> + </distributionManagement> + + <scm> + <connection>scm:git:git@github.com:yahoo/vespa.git</connection> + <developerConnection>scm:git:git@github.com:yahoo/vespa.git</developerConnection> + <url>git@github.com:yahoo/vespa.git</url> + </scm> + + <build> + <finalName>${project.artifactId}</finalName> + <extensions> + <extension> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-ssh-external</artifactId> + <version>2.7</version> + </extension> + <extension> + <groupId>org.apache.maven.archetype</groupId> + <artifactId>archetype-packaging</artifactId> + <version>2.0</version> + </extension> + </extensions> + <pluginManagement> + <plugins> + <plugin> + <groupId>com.github.goldin</groupId> + <artifactId>copy-maven-plugin</artifactId> + <version>0.2.5</version> + </plugin> + <plugin> + <groupId>com.infradna.tool</groupId> + <artifactId>bridge-method-injector</artifactId> + <version>1.4</version> + </plugin> + <plugin> + <groupId>org.antlr</groupId> + <artifactId>antlr3-maven-plugin</artifactId> + <version>3.5.2</version> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <version>2.4.0</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-archetype-plugin</artifactId> + <version>2.0</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + <showWarnings>true</showWarnings> + <optimize>true</optimize> + <showDeprecation>false</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.5</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <version>2.3.1</version> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <configuration> + <additionalparam>-Xdoclint:${doclint} -Xdoclint:-missing</additionalparam> + </configuration> + <version>2.9</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.3</version> + <configuration> + <!-- see http://jira.codehaus.org/browse/MNG-5346 --> + <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> + </configuration> + <executions> + <execution> + <id>mojo-descriptor</id> + <goals> + <goal>descriptor</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-project-info-reports-plugin</artifactId> + <version>2.7</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.5</version> + <configuration> + <escapeString>\</escapeString> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>2.1.2</version> + <configuration> + <includePom>true</includePom> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.16</version> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <systemPropertyVariables> + <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> + </systemPropertyVariables> + <excludes> + <exclude>**/*SystemTest.java</exclude> + </excludes> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>2.16</version> + <configuration> + <alwaysGenerateSurefireReport>false</alwaysGenerateSurefireReport> + <showSuccess>false</showSuccess> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9.1</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <version>2.6</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>properties-maven-plugin</artifactId> + <version>1.0-alpha-2</version> + </plugin> + <plugin> + <groupId>org.scala-tools</groupId> + <artifactId>maven-scala-plugin</artifactId> + <version>2.15.2</version> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${project.version}</version> + <configuration> + <configGenVersion>${project.version}</configGenVersion> + <useCommonAssemblyIds>true</useCommonAssemblyIds> + </configuration> + </plugin> + + <!-- Eclipse-specific stuff, wrong place, but only option, see https://bugs.eclipse.org/bugs/show_bug.cgi?id=350414 --> + <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.--> + <plugin> + <groupId>org.eclipse.m2e</groupId> + <artifactId>lifecycle-mapping</artifactId> + <version>1.0.0</version> + <configuration> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.scala-tools</groupId> + <artifactId>maven-scala-plugin</artifactId> + <versionRange>[2.15.2,)</versionRange> + <goals> + <goal>add-source</goal> + <goal>testCompile</goal> + <goal>compile</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo</groupId> + <artifactId>properties-maven-plugin</artifactId> + <versionRange>[1.0-alpha-2,)</versionRange> + <goals> + <goal>write-project-properties</goal> + <goal>read-project-properties</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <versionRange>[1.1,)</versionRange> + <goals> + <goal>java</goal> + <goal>exec</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <versionRange>[1.3,)</versionRange> + <goals> + <goal>run</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <versionRange>[2.4,)</versionRange> + <goals> + <goal>javacc</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId> + org.apache.maven.plugins + </groupId> + <artifactId> + maven-jar-plugin + </artifactId> + <versionRange> + [0,) + </versionRange> + <goals> + <goal>jar</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId> + com.yahoo.vespa + </groupId> + <artifactId> + container-maven-plugin + </artifactId> + <versionRange> + [0,) + </versionRange> + <goals> + <goal>generateSources</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.antlr</groupId> + <artifactId> + antlr3-maven-plugin + </artifactId> + <versionRange>[0,)</versionRange> + <goals> + <goal>antlr</goal> + </goals> + </pluginExecutionFilter> + <action> + <execute> + <runOnIncremental>false</runOnIncremental> + <runOnConfiguration>true</runOnConfiguration> + </execute> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + <profiles> + <profile> + <id>attach-sources</id> + <activation> + <property> + <name>!skipSources</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <executions> + <execution> + <id>attach-sources</id> + <goals> + <goal>jar-no-fork</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>generate-javadoc</id> + <activation> + <property> + <name>!skipJavadoc</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <id>generate-javadoc</id> + <phase>package</phase> + <goals> + <goal>javadoc</goal> + </goals> + </execution> + </executions> + <configuration> + <additionalparam>-Xdoclint:${doclint} -Xdoclint:-missing</additionalparam> + <failOnError>${javadoc.failOnError}</failOnError> + <quiet>true</quiet> + <show>private</show> + </configuration> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>coverage</id> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <configuration> + <includePluginDependencies>true</includePluginDependencies> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/scala</source> + </sources> + </configuration> + </execution> + <execution> + <id>add-test-source</id> + <phase>generate-test-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>src/test/scala</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>systemtests</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <excludes> + <exclude>none</exclude> + </excludes> + <includes> + <include>**/*SystemTest.java</include> + </includes> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-ssh-external</artifactId> + <version>2.7</version> + </dependency> + <dependency> + <groupId>com.github.cverges.expect4j</groupId> + <artifactId>expect4j</artifactId> + <version>1.6</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <version>1.8</version> + </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>airline</artifactId> + <version>0.7</version> + </dependency> + <dependency> + <groupId>aopalliance</groupId> + <artifactId>aopalliance</artifactId> + <version>1.0</version> + </dependency> + <dependency> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + <version>3.3.1</version> + </dependency> + <dependency> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + <version>5.0.3</version> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>annotations</artifactId> + <version>1.3.9</version> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + <version>1.3.9</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>18.0</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava-testlib</artifactId> + <version>18.0</version> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <version>3.0</version> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <version>3.0</version> + <classifier>no_aop</classifier> + </dependency> + <dependency> + <groupId>com.google.inject.extensions</groupId> + <artifactId>guice-assistedinject</artifactId> + <version>3.0</version> + </dependency> + <dependency> + <groupId>com.google.inject.extensions</groupId> + <artifactId>guice-multibindings</artifactId> + <version>3.0</version> + </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <version>2.4.1</version> + </dependency> + <dependency> + <groupId>com.googlecode.jmockit</groupId> + <artifactId>jmockit</artifactId> + <version>1.2</version> + </dependency> + <dependency> + <groupId>com.goldmansachs</groupId> + <artifactId>gs-collections</artifactId> + <version>6.1.0</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.module</groupId> + <artifactId>jackson-module-jaxb-annotations</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-base</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-xml-provider</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jdk8</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> + <groupId>com.infradna.tool</groupId> + <artifactId>bridge-method-annotation</artifactId> + <version>1.4</version> + </dependency> + <dependency> + <groupId>com.ning</groupId> + <artifactId>async-http-client</artifactId> + <version>1.7.17</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>com.sun.jersey.contribs</groupId> + <artifactId>jersey-guice</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>com.sun.jersey.contribs</groupId> + <artifactId>jersey-multipart</artifactId> + <version>1.13</version> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>1.3.1</version> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.4</version> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>3.2.1</version> + </dependency> + <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + <version>1.6</version> + </dependency> + <dependency> + <groupId>commons-daemon</groupId> + <artifactId>commons-daemon</artifactId> + <version>1.0.3</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.4</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.6</version> + </dependency> + <dependency> + <groupId>commons-net</groupId> + <artifactId>commons-net</artifactId> + <version>2.0</version> + </dependency> + <dependency> + <groupId>commons-pool</groupId> + <artifactId>commons-pool</artifactId> + <version>1.5.6</version> + </dependency> + <!-- Explicitly included to get Zookeeper version 3.4.8, + can be excluded if you want the Zookeeper version + used by curator by default + --> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.4.8</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <!-- WARNING: If you change this version, you also need to update + zkfacade/src/main/java/org/apache/curator/**/package-info.java + using something like + find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ + xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 2, minor = 9, micro = 1/g' + --> + <version>${curator.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>${curator.version}</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> + <version>3.6.5.Final</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + <version>3.1.0</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit-dep</artifactId> + <version>4.11</version> + </dependency> + <dependency> + <groupId>org.antlr</groupId> + <artifactId>antlr-runtime</artifactId> + <version>3.5.2</version> + </dependency> + <dependency> + <groupId>org.apache.aries.spifly</groupId> + <artifactId>org.apache.aries.spifly.dynamic.bundle</artifactId> + <version>${aries.spifly.version}</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.1</version> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + <version>4.2.1</version> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.log</artifactId> + <version>1.0.1</version> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.main</artifactId> + <version>4.2.1</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>fluent-hc</artifactId> + <version>4.3.6</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>4.3.6</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>4.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <version>4.3.6</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-artifact</artifactId> + <version>3.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <version>3.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-model</artifactId> + <version>3.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.2</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-project</artifactId> + <version>2.2.1</version> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-interactivity-api</artifactId> + <version>1.0-alpha-5</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.2</version> + </dependency> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit4</artifactId> + <version>2.16</version> + </dependency> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-providers</artifactId> + <version>2.16</version> + <type>pom</type> + </dependency> + <dependency> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + <version>1.3.1</version> + </dependency> + <dependency> + <groupId>org.cthul</groupId> + <artifactId>cthul-matchers</artifactId> + <version>1.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlets</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty.websocket</groupId> + <artifactId>websocket-server</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty.websocket</groupId> + <artifactId>websocket-servlet</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.grizzly</groupId> + <artifactId>grizzly-websockets</artifactId> + <version>2.3.2</version> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <version>1.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> + <version>1.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-library</artifactId> + <version>1.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>uk.co.datumedge</groupId> + <artifactId>hamcrest-json</artifactId> + <version>0.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <version>20090211</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>1.9.5</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>1.9.5</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.compendium</artifactId> + <version>4.3.0</version> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.core</artifactId> + <version>4.3.0</version> + </dependency> + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-compiler</artifactId> + <version>${scala.version}</version> + </dependency> + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + <version>${scala.version}</version> + </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-parser-combinators_${scala.major-version}</artifactId> + <version>1.0.1</version> + </dependency> + <dependency> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-xml_${scala.major-version}</artifactId> + <version>1.0.2</version> + </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.major-version}</artifactId> + <version>2.2.2</version> + </dependency> + <dependency> + <groupId>org.json4s</groupId> + <artifactId>json4s-native_${scala.major-version}</artifactId> + <version>3.3.0</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <version>4.0.6.RELEASE</version> + </dependency> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>6.9.10</version> + </dependency> + <dependency> + <groupId>org.twdata.maven</groupId> + <artifactId>mojo-executor</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>net.jcip</groupId> + <artifactId>jcip-annotations</artifactId> + <version>1.0</version> + </dependency> + <dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>net.spy</groupId> + <artifactId>spymemcached</artifactId> + <version>2.10.1</version> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + <version>2.11.0</version> + </dependency> + <!-- jersey 2 support --> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <version>${javax.ws.rs-api.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet-core</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-json-jackson</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-multipart</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.ext</groupId> + <artifactId>jersey-proxy-client</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-client</artifactId> + <version>${jersey2.version}</version> + </dependency> + <dependency> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + <version>57.1</version> + </dependency> + </dependencies> + </dependencyManagement> + + <properties> + <javax.ws.rs-api.version>2.0</javax.ws.rs-api.version> + <aries.spifly.version>1.0.8</aries.spifly.version> + <aries.util.version>1.0.0</aries.util.version> + <asm-debug-all.version>5.0.3</asm-debug-all.version> + <curator.version>2.9.1</curator.version> + <jackson2.version>2.5.3</jackson2.version> + <jersey2.version>2.10.1</jersey2.version> + <jetty.version>9.3.8.v20160314</jetty.version> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <test.hide>true</test.hide> + <doclint>all</doclint> + <scala.major-version>2.11</scala.major-version> + <scala.version>${scala.major-version}.4</scala.version> + </properties> + + <modules> + <module>annotations</module> + <module>application</module> + <module>application-deploy-plugin</module> + <module>application-model</module> + <module>application-preprocessor</module> + <module>bundle-plugin</module> + <module>chain</module> + <module>clustercontroller-apps</module> + <module>clustercontroller-apputil</module> + <module>clustercontroller-core</module> + <module>clustercontroller-standalone</module> + <module>clustercontroller-utils</module> + <module>component</module> + <module>config-application-package</module> + <module>configdefinitions</module> + <module>configgen</module> + <module>config-bundle</module> + <module>config-class-plugin</module> + <module>config-lib</module> + <module>config-model-api</module> + <module>config-model-fat</module> + <module>config-model</module> + <module>config</module> + <module>config-provisioning</module> + <module>config-proxy</module> + <module>configserver</module> + <module>config_test</module> + <module>container-core</module> + <module>container-accesslogging</module> + <module>container-dev</module> + <module>container-di</module> + <module>container-disc</module> + <module>container-jersey2</module> + <module>container-messagebus</module> + <module>container-search-and-docproc</module> + <module>container-search</module> + <module>container-test-jars</module> + <module>defaults</module> + <module>docproc</module> + <module>docprocs</module> + <module>documentapi</module> + <module>document</module> + <module>documentgen-test</module> + <module>dummy-persistence</module> + <module>fileacquirer</module> + <module>filedistributionmanager</module> + <module>filedistribution</module> + <module>filedistribution_test</module> + <module>fsa</module> + <module>indexinglanguage</module> + <module>jaxrs_client_utils</module> + <module>jaxrs_utils</module> + <module>jdisc_container_maven_archetype_application</module> + <module>jdisc_core</module> + <module>jdisc_core_test</module> + <module>jdisc_http_service</module> + <module>jdisc_jetty</module> + <module>jdisc_jmx_metrics</module> + <module>jdisc_maven_archetype_component</module> + <module>jdisc_messagebus_service</module> + <module>jrt</module> + <module>libmlr</module> + <module>lifecycle_mapping</module> + <module>linguistics</module> + <module>logd</module> + <module>logserver</module> + <module>messagebus-disc</module> + <module>messagebus</module> + <module>metrics</module> + <module>node-repository</module> + <module>node-admin</module> + <module>orchestrator-restapi</module> + <module>orchestrator</module> + <module>persistence</module> + <module>predicate-search</module> + <module>predicate-search-core</module> + <module>processing</module> + <module>provided-dependencies</module> + <module>searchcore</module> + <module>searchlib</module> + <module>searchsummary</module> + <module>serviceview</module> + <module>service-monitor</module> + <module>simplemetrics</module> + <module>socket_test</module> + <module>standalone-container</module> + <module>statistics</module> + <module>storage</module> + <module>testutil</module> + <module>vdslib</module> + <module>vespaclient-core</module> + <module>vespaclient-container-plugin</module> + <module>vespa-application-maven-plugin</module> + <module>vespa-documentgen-plugin</module> + <module>vespa_feed_perf</module> + <module>vespa-http-client</module> + <module>vespa_jersey2</module> + <module>vespajlib</module> + <module>vespalog</module> + <module>vsm</module> + <module>yolean</module> + <module>zkfacade</module> + <module>scalalib</module> + </modules> </project> diff --git a/predicate-search-core/pom.xml b/predicate-search-core/pom.xml index ad5d6002b66..546d1f5b514 100644 --- a/predicate-search-core/pom.xml +++ b/predicate-search-core/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search-core</artifactId> <version>6-SNAPSHOT</version> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index fe81578c4d7..9906a46654b 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search</artifactId> <version>6-SNAPSHOT</version> diff --git a/processing/pom.xml b/processing/pom.xml index 3cc44b66e05..a5f25d0aa0f 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>processing</artifactId> <packaging>jar</packaging> diff --git a/provided-dependencies/pom.xml b/provided-dependencies/pom.xml index 332fff2a62f..0300ef1d9c4 100755 --- a/provided-dependencies/pom.xml +++ b/provided-dependencies/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>provided-dependencies</artifactId> <packaging>jar</packaging> diff --git a/sample-apps/README.md b/sample-apps/README.md new file mode 100644 index 00000000000..22500f9ecb5 --- /dev/null +++ b/sample-apps/README.md @@ -0,0 +1,5 @@ +# Vespa sample applications +This is the future home of Vespa Sample applications + +## TODO: +* Add basic-search diff --git a/sample-apps/basic-search/README.md b/sample-apps/basic-search/README.md new file mode 100644 index 00000000000..616c10acab0 --- /dev/null +++ b/sample-apps/basic-search/README.md @@ -0,0 +1,4 @@ +# Vespa sample applications - Basic Search +Set up a simple Vespa application on one node. +Feed and run simple queries + diff --git a/sample-apps/basic-search/music-data-1.json b/sample-apps/basic-search/music-data-1.json new file mode 100644 index 00000000000..44ce547370e --- /dev/null +++ b/sample-apps/basic-search/music-data-1.json @@ -0,0 +1,9 @@ +{ + "fields": { + "album": "Bad", + "artist": "Michael Jackson", + "title": "Bad", + "year": 1987, + "duration": 247 + } +} diff --git a/sample-apps/basic-search/music-data-2.json b/sample-apps/basic-search/music-data-2.json new file mode 100644 index 00000000000..c8e2d3c017c --- /dev/null +++ b/sample-apps/basic-search/music-data-2.json @@ -0,0 +1,9 @@ +{ + "fields": { + "album": "Recovery", + "artist": "Eminem", + "title": "So Bad", + "year": 2010 + } +} + diff --git a/sample-apps/basic-search/music-data-feed.json b/sample-apps/basic-search/music-data-feed.json new file mode 100644 index 00000000000..f38cbef9136 --- /dev/null +++ b/sample-apps/basic-search/music-data-feed.json @@ -0,0 +1,22 @@ +[ + { + "put": "id:music:music::Michael-Jackson-Bad", + "fields": { + "album": "Bad", + "artist": "Michael Jackson", + "title": "Bad", + "year": 1987, + "duration": 247 + } + }, + { + "put": "id:music:music::Eminem-Recovery", + "fields": { + "album": "Recovery", + "artist": "Eminem", + "title": "So Bad", + "year": 2010 + } + } +] + diff --git a/sample-apps/basic-search/src/main/application/hosts.xml b/sample-apps/basic-search/src/main/application/hosts.xml new file mode 100644 index 00000000000..632e48db321 --- /dev/null +++ b/sample-apps/basic-search/src/main/application/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> + diff --git a/sample-apps/basic-search/src/main/application/searchdefinitions/music.sd b/sample-apps/basic-search/src/main/application/searchdefinitions/music.sd new file mode 100644 index 00000000000..d6bcaac8ae0 --- /dev/null +++ b/sample-apps/basic-search/src/main/application/searchdefinitions/music.sd @@ -0,0 +1,40 @@ +search music { + document music { + field artist type string { + indexing: summary | index + } + field artistId type string { + indexing: summary | attribute + } + + field title type string { + indexing: summary | index + } + + field album type string { + indexing: index + } + + field duration type int { + indexing: summary + } + + field year type int { + indexing: summary | attribute + } + + field popularity type int { + indexing: summary | attribute + } + } + + fieldset default { + fields: artist, title, album + } + + rank-profile song inherits default { + first-phase { + expression:nativeRank(artist,title,album) + if(isNan(attribute(popularity)) == 1, 0,attribute(popularity)) + } + } +} diff --git a/vespaapplication/examples/music/services.xml b/sample-apps/basic-search/src/main/application/services.xml index 8627e83e6c9..69899bcd1c6 100644 --- a/vespaapplication/examples/music/services.xml +++ b/sample-apps/basic-search/src/main/application/services.xml @@ -1,12 +1,8 @@ <?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> - <admin version="2.0"> - <adminserver hostalias="node1"/> - </admin> - <jdisc version="1.0" id="container"> + <document-api /> <search /> <nodes> <node hostalias="node1" /> @@ -24,3 +20,4 @@ </content> </services> + diff --git a/scalalib/pom.xml b/scalalib/pom.xml index a9b0cade639..94706777f8d 100644 --- a/scalalib/pom.xml +++ b/scalalib/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>scalalib</artifactId> <packaging>jar</packaging> diff --git a/searchcommon/.gitignore b/searchcommon/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/searchcommon/.gitignore +++ b/searchcommon/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/searchcommon/OWNERS b/searchcommon/OWNERS index 593ba19b1e6..cf65e9c8229 100644 --- a/searchcommon/OWNERS +++ b/searchcommon/OWNERS @@ -1,3 +1,3 @@ geirst baldersheim -tegge +toregge diff --git a/searchcommon/src/tests/schema/.gitignore b/searchcommon/src/tests/schema/.gitignore index 79e714aa5a2..ae6d9596934 100644 --- a/searchcommon/src/tests/schema/.gitignore +++ b/searchcommon/src/tests/schema/.gitignore @@ -2,3 +2,7 @@ /Makefile /schema_test searchcommon_schema_test_app +/schema-with-timestamps.txt +/schema-without-timestamps.txt +/schema.txt +/schema2.txt diff --git a/searchcore/.gitignore b/searchcore/.gitignore index bb01e126e8c..6f79fc39b2d 100644 --- a/searchcore/.gitignore +++ b/searchcore/.gitignore @@ -12,4 +12,3 @@ update.log /target /pom.xml.build Makefile -Testing diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index fa5576a6612..3d72d70af48 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -54,9 +54,6 @@ vespa_define_module( src/apps/vespa-proton-cmd src/apps/vespa-transactionlog-inspect - APP_DEPENDS - persistence_persistence_conformancetest - TESTS src/tests/applyattrupdates src/tests/fdispatch/randomrow diff --git a/searchcore/OWNERS b/searchcore/OWNERS index 0775e8692a2..81de3c9fc79 100644 --- a/searchcore/OWNERS +++ b/searchcore/OWNERS @@ -1,5 +1,5 @@ geirst -tegge +toregge baldersheim havardpe yngve diff --git a/searchcore/pom.xml b/searchcore/pom.xml index 6654c75ba26..04d9f3d0021 100644 --- a/searchcore/pom.xml +++ b/searchcore/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchcore</artifactId> <version>6-SNAPSHOT</version> diff --git a/searchcore/src/apps/tests/CMakeLists.txt b/searchcore/src/apps/tests/CMakeLists.txt index 37287fdda37..4bb704982d3 100644 --- a/searchcore/src/apps/tests/CMakeLists.txt +++ b/searchcore/src/apps/tests/CMakeLists.txt @@ -21,5 +21,5 @@ vespa_add_executable(searchcore_persistenceconformance_test_app TEST searchcore_fconfig searchcore_util vdstestlib + persistence_persistence_conformancetest ) -vespa_add_test(NAME searchcore_persistenceconformance_test_app COMMAND searchcore_persistenceconformance_test_app) diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index e0d92cd2a1a..0b59a3c11ed 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -106,14 +106,15 @@ struct UnitDR : DocumentRetrieverBaseForTest { Bucket bucket; bool removed; DocumentIdT docid; + DocumentIdT docIdLimit; UnitDR() : repo(), document(new Document(*DataType::DOCUMENT, DocumentId())), - timestamp(0), bucket(), removed(false), docid(0) {} + timestamp(0), bucket(), removed(false), docid(0), docIdLimit(std::numeric_limits<uint32_t>::max()) {} UnitDR(document::Document::UP d, Timestamp t, Bucket b, bool r) - : repo(), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt) {} + : repo(), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt), docIdLimit(std::numeric_limits<uint32_t>::max()) {} UnitDR(const document::DocumentType &dt, document::Document::UP d, Timestamp t, Bucket b, bool r) - : repo(dt), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt) {} + : repo(dt), document(std::move(d)), timestamp(t), bucket(b), removed(r), docid(++_docidCnt), docIdLimit(std::numeric_limits<uint32_t>::max()) {} const document::DocumentTypeRepo &getDocumentTypeRepo() const override { return repo; @@ -134,6 +135,13 @@ struct UnitDR : DocumentRetrieverBaseForTest { return Document::UP((lid == docid) ? document->clone() : 0); } + uint32_t getDocIdLimit() const override { + return docIdLimit; + } + void setDocIdLimit(DocumentIdT limit) { + docIdLimit = limit; + } + CachedSelect::SP parseSelect(const vespalib::string &selection) const override { CachedSelect::SP res(new CachedSelect); res->set(selection, repo); @@ -509,12 +517,6 @@ void verifyStrongReadConsistency(DocumentIterator & itr) { EXPECT_EQUAL(1u, committer._commitAndWaitCount); } -void verifyWeakReadConsistency(DocumentIterator & itr) { - Committer committer; - TEST_DO(verifyReadConsistency(itr, committer)); - EXPECT_EQUAL(0u, committer._commitAndWaitCount); -} - TEST("require that default readconsistency does commit") { DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); TEST_DO(verifyStrongReadConsistency(itr)); @@ -525,9 +527,23 @@ TEST("require that readconsistency::strong does commit") { TEST_DO(verifyStrongReadConsistency(itr)); } -TEST("require that readconsistency::weak does not commit") { - DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false, storage::spi::ReadConsistency::WEAK); - TEST_DO(verifyWeakReadConsistency(itr)); +TEST("require that docid limit is honoured") { + IDocumentRetriever::SP retriever = doc("doc:foo:1", Timestamp(2), bucket(5)); + UnitDR & udr = dynamic_cast<UnitDR &>(*retriever); + udr.docid = 7; + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); + itr.add(retriever); + IterateResult res = itr.iterate(largeNum); + EXPECT_TRUE(res.isCompleted()); + EXPECT_EQUAL(1u, res.getEntries().size()); + TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("doc:foo:1")), Timestamp(2))); + + udr.setDocIdLimit(7); + DocumentIterator limited(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); + limited.add(retriever); + res = limited.iterate(largeNum); + EXPECT_TRUE(res.isCompleted()); + EXPECT_EQUAL(0u, res.getEntries().size()); } TEST("require that remove entries can be iterated") { diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 1764d6f2996..6b6abe5bc09 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -135,6 +135,7 @@ struct Fixture _views(), _configurer() { + vespalib::rmdir(BASE_DIR, true); vespalib::mkdir(BASE_DIR); initViewSet(_views); _configurer.reset(new Configurer(_views._summaryMgr, @@ -146,7 +147,6 @@ struct Fixture 0)); } ~Fixture() { - vespalib::rmdir(BASE_DIR, true); } void initViewSet(ViewSet &views); }; @@ -293,11 +293,11 @@ struct FastAccessFixture _configurer(_view._feedView, IAttributeAdapterFactory::UP(new AttributeAdapterFactory), "test") { + vespalib::rmdir(BASE_DIR, true); vespalib::mkdir(BASE_DIR); } ~FastAccessFixture() { _writeService.sync(); - vespalib::rmdir(BASE_DIR, true); } }; @@ -608,4 +608,5 @@ TEST_F("require that we can reconfigure matchers", Fixture) TEST_MAIN() { TEST_RUN_ALL(); + vespalib::rmdir(BASE_DIR, true); } diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index e1e9f58fc14..7b09beb35b6 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -44,6 +44,15 @@ using proton::bucketdb::BucketState; namespace proton { +namespace +{ + +static constexpr uint32_t numBucketBits = UINT32_C(20); +static constexpr uint64_t timestampBias = UINT64_C(2000000000000); + +} + + class DummyTlsSyncer : public ITlsSyncer { public: @@ -552,23 +561,21 @@ TEST("requireThatWeCanStoreBucketIdAndTimestamp") { DocumentMetaStore dms(createBucketDB()); uint32_t numLids = 1000; - uint32_t bkBits = UINT32_C(20); - uint64_t tsbias = UINT64_C(2000000000000); dms.constructFreeList(); for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); - uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + tsbias)); + bucketId.setUsedBits(numBucketBits); + uint32_t addLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); EXPECT_EQUAL(lid, addLid); } for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); + bucketId.setUsedBits(numBucketBits); EXPECT_TRUE(assertGid(gid, lid, dms, bucketId, - Timestamp(lid + tsbias))); + Timestamp(lid + timestampBias))); EXPECT_TRUE(assertLid(lid, gid, dms)); } } @@ -577,8 +584,6 @@ TEST("requireThatGidsCanBeSavedAndLoaded") { DocumentMetaStore dms1(createBucketDB()); uint32_t numLids = 1000; - uint32_t bkBits = UINT32_C(20); - uint64_t tsbias = UINT64_C(2000000000000); std::vector<uint32_t> removeLids; removeLids.push_back(10); removeLids.push_back(20); @@ -588,8 +593,8 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); - uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + tsbias)); + bucketId.setUsedBits(numBucketBits); + uint32_t addLid = addGid(dms1, gid, bucketId, Timestamp(lid + timestampBias)); EXPECT_EQUAL(lid, addLid); } for (size_t i = 0; i < removeLids.size(); ++i) { @@ -612,10 +617,10 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (uint32_t lid = 1; lid <= numLids; ++lid) { GlobalId gid = createGid(lid); BucketId bucketId(gid.convertToBucketId()); - bucketId.setUsedBits(bkBits); + bucketId.setUsedBits(numBucketBits); if (std::count(removeLids.begin(), removeLids.end(), lid) == 0) { EXPECT_TRUE(assertGid(gid, lid, dms2, bucketId, - Timestamp(lid + tsbias))); + Timestamp(lid + timestampBias))); EXPECT_TRUE(assertLid(lid, gid, dms2)); } else { LOG(info, "Lid %u was removed before saving", lid); @@ -629,7 +634,7 @@ TEST("requireThatGidsCanBeSavedAndLoaded") for (size_t i = 0; i < removeLids.size(); ++i) { LOG(info, "Re-use remove lid %u", removeLids[i]); GlobalId gid = createGid(removeLids[i]); - BucketId bucketId(bkBits, + BucketId bucketId(numBucketBits, gid.convertToBucketId().getRawId()); // re-use removeLid[i] uint32_t addLid = addGid(dms2, gid, bucketId, Timestamp(43u + i)); @@ -1870,6 +1875,69 @@ TEST("requireThatShrinkViaFlushTargetWorks") ft->getApproxMemoryGain().getAfter()); } + +namespace { + +void +addLid(DocumentMetaStore &dms, uint32_t lid) +{ + GlobalId gid = createGid(lid); + BucketId bucketId(gid.convertToBucketId()); + bucketId.setUsedBits(numBucketBits); + uint32_t addedLid = addGid(dms, gid, bucketId, Timestamp(lid + timestampBias)); + EXPECT_EQUAL(lid, addedLid); +} + +void +removeLid(DocumentMetaStore &dms, uint32_t lid) +{ + dms.remove(lid); + dms.removeComplete(lid); +} + + +void +assertCompact(DocumentMetaStore &dms, uint32_t docIdLimit, + uint32_t committedDocIdLimit, + uint32_t compactTarget, uint32_t numUsedLids) +{ + EXPECT_TRUE(assertLidSpace(docIdLimit, committedDocIdLimit, numUsedLids, false, false, dms)); + dms.compactLidSpace(compactTarget); + EXPECT_TRUE(assertLidSpace(docIdLimit, compactTarget, numUsedLids, true, false, dms)); + dms.holdUnblockShrinkLidSpace(); + EXPECT_TRUE(assertLidSpace(docIdLimit, compactTarget, numUsedLids, true, true, dms)); +} + + +void +assertShrink(DocumentMetaStore &dms, uint32_t shrinkTarget, + uint32_t numUsedLids) +{ + dms.shrinkLidSpace(); + TEST_DO(EXPECT_TRUE(assertLidSpace(shrinkTarget, shrinkTarget, numUsedLids, false, false, dms))); +} + +} + + +TEST("requireThatSecondShrinkWorksAfterCompactAndInactiveInsert") +{ + DocumentMetaStore dms(createBucketDB()); + dms.constructFreeList(); + TEST_DO(addLid(dms, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(addLid(dms, 3)); + removeLid(dms, 2); + removeLid(dms, 3); + EXPECT_TRUE(assertLidSpace(4, 4, 1, false, false, dms)); + TEST_DO(assertCompact(dms, 4, 4, 2, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(assertShrink(dms, 3, 2)); + removeLid(dms, 2); + TEST_DO(assertCompact(dms, 3, 3, 2, 1)); + TEST_DO(assertShrink(dms, 2, 1)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/flushengine/flushengine.cpp b/searchcore/src/tests/proton/flushengine/flushengine.cpp index 59b86671a0d..add89d5fc58 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine.cpp @@ -15,6 +15,7 @@ LOG_SETUP("flushengine_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/test/insertion_operators.h> #include <memory> // -------------------------------------------------------------------------------- @@ -67,14 +68,63 @@ class SimpleTlsStatsFactory : public flushengine::ITlsStatsFactory } }; +class SimpleHandler; + +class WrappedFlushTask : public searchcorespi::FlushTask +{ + searchcorespi::FlushTask::UP _task; + SimpleHandler &_handler; + +public: + virtual void run() override; + WrappedFlushTask(searchcorespi::FlushTask::UP task, + SimpleHandler &handler) + : _task(std::move(task)), + _handler(handler) + { + } + + virtual search::SerialNum getFlushSerial() const override + { + return _task->getFlushSerial(); + } +}; + +class WrappedFlushTarget : public FlushTargetProxy +{ + SimpleHandler &_handler; +public: + WrappedFlushTarget(const IFlushTarget::SP &target, + SimpleHandler &handler) + : FlushTargetProxy(target), + _handler(handler) + { + } + + virtual Task::UP initFlush(SerialNum currentSerial) override + { + Task::UP task(_target->initFlush(currentSerial)); + if (task) { + return std::make_unique<WrappedFlushTask>(std::move(task), + _handler); + } + return std::move(task); + } +}; + typedef std::vector<IFlushTarget::SP> Targets; +using FlushDoneHistory = std::vector<search::SerialNum>; + class SimpleHandler : public test::DummyFlushHandler { public: Targets _targets; search::SerialNum _oldestSerial; search::SerialNum _currentSerial; + uint32_t _pendingDone; + vespalib::Lock _lock; vespalib::CountDownLatch _done; + FlushDoneHistory _flushDoneHistory; public: typedef std::shared_ptr<SimpleHandler> SP; @@ -85,7 +135,10 @@ public: _targets(targets), _oldestSerial(0), _currentSerial(currentSerial), - _done(targets.size()) + _pendingDone(0u), + _lock(), + _done(targets.size()), + _flushDoneHistory() { // empty } @@ -103,21 +156,54 @@ public: { LOG(info, "SimpleHandler(%s)::getFlushTargets()", getName().c_str()); - return _targets; + std::vector<IFlushTarget::SP> wrappedTargets; + for (const auto &target : _targets) { + wrappedTargets.push_back(std::make_shared<WrappedFlushTarget> + (target, *this)); + } + return std::move(wrappedTargets); + } + + // Called once by flush engine slave thread for each task done + void taskDone() + { + vespalib::LockGuard guard(_lock); + ++_pendingDone; } + // Called by flush engine master thread after flush handler is + // added to flush engine and when one or more flush tasks related + // to flush handler have completed. void flushDone(search::SerialNum oldestSerial) override { + vespalib::LockGuard guard(_lock); LOG(info, "SimpleHandler(%s)::flushDone(%" PRIu64 ")", getName().c_str(), oldestSerial); _oldestSerial = std::max(_oldestSerial, oldestSerial); - _done.countDown(); - } + _flushDoneHistory.push_back(oldestSerial); + while (_pendingDone > 0) { + --_pendingDone; + _done.countDown(); + } + } + FlushDoneHistory getFlushDoneHistory() + { + vespalib::LockGuard guard(_lock); + return _flushDoneHistory; + } }; +void WrappedFlushTask::run() +{ + _task->run(); + _handler.taskDone(); +} + class SimpleTask : public searchcorespi::FlushTask { + search::SerialNum &_flushedSerial; + search::SerialNum &_currentSerial; public: vespalib::Gate &_start; vespalib::Gate &_done; @@ -126,8 +212,11 @@ public: public: SimpleTask(vespalib::Gate &start, vespalib::Gate &done, - vespalib::Gate *proceed) - : _start(start), _done(done), _proceed(proceed) + vespalib::Gate *proceed, + search::SerialNum &flushedSerial, + search::SerialNum ¤tSerial) + : _flushedSerial(flushedSerial), _currentSerial(currentSerial), + _start(start), _done(done), _proceed(proceed) { // empty } @@ -137,6 +226,7 @@ public: if (_proceed != NULL) { _proceed->await(); } + _flushedSerial = _currentSerial; _done.countDown(); } @@ -150,6 +240,7 @@ public: class SimpleTarget : public test::DummyFlushTarget { public: search::SerialNum _flushedSerial; + search::SerialNum _currentSerial; vespalib::Gate _proceed; vespalib::Gate _initDone; vespalib::Gate _taskStart; @@ -162,6 +253,7 @@ public: SimpleTarget(Task::UP task, const std::string &name) : test::DummyFlushTarget(name), _flushedSerial(0), + _currentSerial(0), _proceed(), _initDone(), _taskStart(), @@ -177,7 +269,8 @@ public: _initDone(), _taskStart(), _taskDone(), - _task(new SimpleTask(_taskStart, _taskDone, &_proceed)) + _task(new SimpleTask(_taskStart, _taskDone, &_proceed, + _flushedSerial, _currentSerial)) { if (proceedImmediately) { _proceed.countDown(); @@ -190,8 +283,8 @@ public: virtual SerialNum getFlushedSerialNum() const override { - LOG(info, "SimpleTarget(%s)::getFlushedSerialNum()", - getName().c_str()); + LOG(info, "SimpleTarget(%s)::getFlushedSerialNum() = %" PRIu64, + getName().c_str(), _flushedSerial); return _flushedSerial; } @@ -200,6 +293,7 @@ public: { LOG(info, "SimpleTarget(%s)::initFlush(%" PRIu64 ")", getName().c_str(), currentSerial); + _currentSerial = currentSerial; _initDone.countDown(); return std::move(_task); } @@ -282,6 +376,10 @@ public: if (cached != NULL) { raw = cached->getFlushTarget().get(); } + WrappedFlushTarget *wrapped = dynamic_cast<WrappedFlushTarget *>(raw); + if (wrapped != nullptr) { + raw = wrapped->getFlushTarget().get(); + } for (uint32_t i = 0, len = _targets.size(); i < len; ++i) { if (raw == _targets[i].get()) { LOG(info, "Index of target %p is %d.", raw, i); @@ -340,7 +438,7 @@ struct Fixture Fixture(uint32_t numThreads, uint32_t idleIntervalMS) : tlsStatsFactory(std::make_shared<SimpleTlsStatsFactory>()), strategy(std::make_shared<SimpleStrategy>()), - engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS, false) + engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS) { } }; @@ -398,7 +496,9 @@ TEST_F("require that oldest serial is found", Fixture(1, IINTERVAL)) f.engine.start(); EXPECT_TRUE(handler->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(20ul, handler->_oldestSerial); + EXPECT_EQUAL(25ul, handler->_oldestSerial); + FlushDoneHistory handlerFlushDoneHistory(handler->getFlushDoneHistory()); + EXPECT_EQUAL(FlushDoneHistory({ 10, 20, 25 }), handlerFlushDoneHistory); } TEST_F("require that oldest serial is found in group", Fixture(2, IINTERVAL)) @@ -423,9 +523,21 @@ TEST_F("require that oldest serial is found in group", Fixture(2, IINTERVAL)) f.engine.start(); EXPECT_TRUE(fooH->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(20ul, fooH->_oldestSerial); + EXPECT_EQUAL(25ul, fooH->_oldestSerial); + // [ 10, 25 ], [ 10, 25, 25 ] and [ 10, 20, 25 ] are legal histories + FlushDoneHistory fooHFlushDoneHistory(fooH->getFlushDoneHistory()); + if (fooHFlushDoneHistory != FlushDoneHistory({ 10, 25 }) && + fooHFlushDoneHistory != FlushDoneHistory({ 10, 25, 25 })) { + EXPECT_EQUAL(FlushDoneHistory({ 10, 20, 25 }), fooHFlushDoneHistory); + } EXPECT_TRUE(barH->_done.await(LONG_TIMEOUT)); - EXPECT_EQUAL(15ul, barH->_oldestSerial); + EXPECT_EQUAL(20ul, barH->_oldestSerial); + // [ 5, 20 ], [ 5, 20, 20 ] and [ 5, 15, 20 ] are legal histories + FlushDoneHistory barHFlushDoneHistory(barH->getFlushDoneHistory()); + if (barHFlushDoneHistory != FlushDoneHistory({ 5, 20 }) && + barHFlushDoneHistory != FlushDoneHistory({ 5, 20, 20 })) { + EXPECT_EQUAL(FlushDoneHistory({ 5, 15, 20 }), barHFlushDoneHistory); + } } TEST_F("require that target can refuse flush", Fixture(2, IINTERVAL)) diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/.gitignore b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/.gitignore new file mode 100644 index 00000000000..d06d6be719a --- /dev/null +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/.gitignore @@ -0,0 +1 @@ +/searchcore_flushengine_prepare_restart_flush_strategy_test_app diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index ac3dbb8fed2..b3fd9a050a8 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -72,7 +72,7 @@ public: targetType, flushedSerial, approxDiskBytes); - _result.push_back(std::make_shared<FlushContext>(handler, target, 0, 0)); + _result.push_back(std::make_shared<FlushContext>(handler, target, 0)); return *this; } ContextsBuilder &add(const vespalib::string &handlerName, diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 4346f7d43c1..b78a3060238 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -273,7 +273,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { handleJoin(FeedToken token, const storage::spi::Bucket &source1, const storage::spi::Bucket &source2, - const storage::spi::Bucket &target) + const storage::spi::Bucket &target) override { (void) source1; (void) source2; @@ -282,7 +282,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { token.ack(); } - virtual RetrieversSP getDocumentRetrievers() { + virtual RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency) override { RetrieversSP ret(new std::vector<IDocumentRetriever::SP>); ret->push_back(IDocumentRetriever::SP(new MyDocumentRetriever( 0, Timestamp(), lastDocId))); @@ -291,12 +291,12 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { return ret; } - virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &b) { + virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &b) override { return BucketGuard::UP(new BucketGuard(b.getBucketId(), *this)); } virtual void - handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) + handleListActiveBuckets(IBucketIdListResultHandler &resultHandler) override { BucketIdListResult::List list; resultHandler.handle(BucketIdListResult(list)); @@ -304,17 +304,17 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { virtual void handlePopulateActiveBuckets(document::BucketId::List &buckets, - IGenericResultHandler &resultHandler) + IGenericResultHandler &resultHandler) override { (void) buckets; resultHandler.handle(Result()); } - virtual void freezeBucket(BucketId bucket) { + virtual void freezeBucket(BucketId bucket) override { frozen.insert(bucket.getId()); was_frozen.insert(bucket.getId()); } - virtual void thawBucket(BucketId bucket) { + virtual void thawBucket(BucketId bucket) override { std::multiset<uint64_t>::iterator it = frozen.find(bucket.getId()); ASSERT_TRUE(it != frozen.end()); frozen.erase(it); diff --git a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp index 2f4083228f9..6c235766de6 100644 --- a/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp +++ b/searchcore/src/tests/proton/server/memoryflush/memoryflush_test.cpp @@ -105,7 +105,7 @@ public: return *this; } ContextBuilder &add(const IFlushTarget::SP &target, SerialNum lastSerial = 0) { - FlushContext::SP ctx(new FlushContext(_handler, target, 0, lastSerial)); + FlushContext::SP ctx(new FlushContext(_handler, target, lastSerial)); return add(ctx); } const FlushContext::List &list() const { return _list; } @@ -282,22 +282,22 @@ requireThatWeCanOrderByTlsSize() (handler1, createTargetT("t2", TimeStamp(now.val() - 10 * TimeStamp::SEC), 1900), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler2, createTargetT("t1", TimeStamp(now.val() - 5 * TimeStamp::SEC), 1000), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler1, createTargetT("t4", TimeStamp(), 1000), - 2000, 2000)). + 2000)). add(std::make_shared<FlushContext> (handler2, createTargetT("t3", TimeStamp(now.val() - 15 * TimeStamp::SEC), 1900), - 2000, 2000)); + 2000)); { // sum of tls sizes above limit, trigger sort order based on tls size MemoryFlush flush({1000, 3 * gibi, 1.0, 1000, 1.0, 2000, TimeStamp(2 * TimeStamp::SEC)}, start); EXPECT_TRUE(assertOrder(StringList().add("t4").add("t1").add("t2").add("t3"), diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index b55f29ff810..89e15952a42 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -57,26 +57,26 @@ flush.idleinterval double default=10.0 restart flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart ## Total number of bytes allowed before forcing flush. -flush.memory.maxmemory long default=4294967296 restart +flush.memory.maxmemory long default=4294967296 ## Maximum total disk bloat factor before forcing flush. -flush.memory.diskbloatfactor double default=0.2 restart +flush.memory.diskbloatfactor double default=0.2 ## Max disk usage (in bytes) for all transaction logs before forcing flush. -flush.memory.maxtlssize long default=21474836480 restart +flush.memory.maxtlssize long default=21474836480 ## Number of bytes allowed per component before forcing memory prioritization. -flush.memory.each.maxmemory long default=1073741824 restart +flush.memory.each.maxmemory long default=1073741824 ## Maximum disk bloat factor per component before forcing flush. -flush.memory.each.diskbloatfactor double default=0.2 restart +flush.memory.each.diskbloatfactor double default=0.2 ## Age of unflushed content before forcing age prioritization. ## Unit is seconds with 1 day being the default. -flush.memory.maxage.time double default=86400.0 restart +flush.memory.maxage.time double default=86400.0 ## Max diff in serial number allowed before that takes precedence. -flush.memory.maxage.serial long default=1000000 restart +flush.memory.maxage.serial long default=1000000 ## The cost of doing replay when replaying the transaction log. ## @@ -86,7 +86,7 @@ flush.memory.maxage.serial long default=1000000 restart ## The prepare for restart flush strategy will choose a set of components to flush ## such that the cost of flushing these + the cost of replaying the transaction log ## is as low as possible. -flush.preparerestart.replaycost double default=4.0 restart +flush.preparerestart.replaycost double default=4.0 ## The cost of doing writes when flushing components to disk. ## @@ -96,7 +96,7 @@ flush.preparerestart.replaycost double default=4.0 restart ## The prepare for restart flush strategy will choose a set of components to flush ## such that the cost of flushing these + the cost of replaying the transaction log ## is as low as possible. -flush.preparerestart.writecost double default=1.0 restart +flush.preparerestart.writecost double default=1.0 ## Control io options during write both under dump and fusion. indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart diff --git a/searchcore/src/vespa/searchcore/fdispatch/OWNERS b/searchcore/src/vespa/searchcore/fdispatch/OWNERS index d14485102db..f4d47806ed9 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/OWNERS +++ b/searchcore/src/vespa/searchcore/fdispatch/OWNERS @@ -1,2 +1,2 @@ baldersheim -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index 193c4365189..6e06c91a902 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -31,18 +31,12 @@ using document::CompressionConfig; char FastS_VersionTag[] = V_TAG; -int FastS_verbose = 0; -/** @todo Use a config file for these variables */ -int FastS_nsearches; -double FastS_searchtime; - - namespace fdispatch { FastS_FNETAdapter::FastS_FNETAdapter(FastS_AppContext *appCtx) : _appCtx(appCtx), - _nodeManager(NULL), + _nodeManager(), _timeKeeper(NULL), _transport(NULL), _last_now(0.0), @@ -92,57 +86,57 @@ FastS_FNETAdapter::fini() Fdispatch::~Fdispatch(void) { - if (_transportServer != NULL) { + if (_transportServer) { _transportServer->shutDown(); // sync shutdown } _FNET_adapter.fini(); - if (_nodeManager != NULL) + if (_nodeManager) { _nodeManager->ShutdownConfig(); - if (_transport != NULL && _transportStarted) + } + if (_transport && _transportStarted) { _transport->ShutDown(true); // sync shutdown - if (_rpc != NULL) + } + if (_rpc) { _rpc->ShutDown(); // sync shutdown + } LOG(debug, "Will close threadpool"); _mypool->Close(); LOG(debug, "Has closed threadpool"); - delete _transportServer; - delete _engineAdapter; - delete _nodeManager; - if (_transport != NULL) { - delete _transport; - } - delete _rpc; - delete _mypool; + _transportServer.reset(); + _engineAdapter.reset(); + _nodeManager.reset(); + _transport.reset(); + _rpc.reset(); + _mypool.reset(); } FNET_Transport * Fdispatch::GetFNETTransport() { - return _transport; + return _transport.get(); } FNET_Scheduler * Fdispatch::GetFNETScheduler() { - return (_transport == NULL) ? - NULL : _transport->GetScheduler(); + return ( ! _transport) ? NULL : _transport->GetScheduler(); } FastS_NodeManager * Fdispatch::GetNodeManager() { - return _nodeManager; + return _nodeManager.get(); } FastS_DataSetCollection * Fdispatch::GetDataSetCollection() { - if (_nodeManager == NULL) + if ( ! _nodeManager) return NULL; return _nodeManager->GetDataSetCollection(); } @@ -151,14 +145,14 @@ Fdispatch::GetDataSetCollection() FastOS_ThreadPool * Fdispatch::GetThreadPool() { - return _mypool; + return _mypool.get(); } bool Fdispatch::Failed(void) { - return ( (_transportServer != NULL && _transportServer->isFailed())); + return ( (_transportServer && _transportServer->isFailed())) || _needRestart; } @@ -204,7 +198,7 @@ Fdispatch::CheckTempFail(void) if (failflag == _tempFail) return ret; - if (_transportServer != NULL) { + if (_transportServer) { if (failflag) { _transportServer->setListen(false); LOG(error, "Disabling fnet server interface"); @@ -223,16 +217,18 @@ Fdispatch::CheckTempFail(void) * Set up stuff as specified in the fdispatch-rc-file. */ Fdispatch::Fdispatch(const config::ConfigUri &configUri) - : _mypool(NULL), - _engineAdapter(NULL), - _transportServer(NULL), + : _mypool(), + _engineAdapter(), + _transportServer(), _componentConfig(), - _nodeManager(NULL), - _transport(NULL), + _nodeManager(), + _transport(), _FNET_adapter(this), - _rpc(NULL), + _rpc(), _config(), _configUri(configUri), + _fdispatchrcFetcher(configUri.getContext()), + _rndGen(), _partition(0), _tempFail(false), _FNETLiveCounterDanger(false), @@ -243,32 +239,51 @@ Fdispatch::Fdispatch(const config::ConfigUri &configUri) _FNETLiveCounterDangerStart(), _timeouts(0u), _checkLimit(0u), - _healthPort(0) + _healthPort(0), + _needRestart(false) { int64_t cfgGen = -1; _config = config::ConfigGetter<FdispatchrcConfig>:: - getConfig(cfgGen, - _configUri.getConfigId(), - _configUri.getContext()); + getConfig(cfgGen, _configUri.getConfigId(), _configUri.getContext()); LOG(config, "fdispatch version %s (RPC-port: %d, transport at %d)", FastS_VersionTag, _config->frtport, _config->ptport); _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("fdispatch", cfgGen, "config only obtained at startup")); + _fdispatchrcFetcher.subscribe<FdispatchrcConfig>(configUri.getConfigId(), this); + _fdispatchrcFetcher.start(); } +namespace { -void -Fdispatch::CheckCacheMaxEntries(unsigned int queryCacheMaxEntries, - unsigned int queryAttrCacheMaxEntries) +bool needRestart(const FdispatchrcConfig & curr, const FdispatchrcConfig & next) { - if (queryAttrCacheMaxEntries == 0) - return; + if (curr.frtport != next.frtport) { + LOG(error, "FRT port has changed from %d to %d.", curr.frtport, next.frtport); + return true; + } + if (curr.ptport != next.ptport) { + LOG(error, "PT port has changed from %d to %d.", curr.ptport, next.ptport); + return true; + } + if (curr.healthport != next.healthport) { + LOG(error, "Health port has changed from %d to %d.", curr.healthport, next.healthport); + return true; + } + return false; +} + +} - if ((queryAttrCacheMaxEntries <= queryCacheMaxEntries) || - (queryCacheMaxEntries == 0)) { - FastS_abort("Please edit fdispatchrc such that " - "queryattrcachequeries > querycachequeries."); +void Fdispatch::configure(std::unique_ptr<FdispatchrcConfig> cfg) +{ + if (cfg && _config) { + if ( needRestart(*_config, *cfg) ) { + const int sleepMS = (1.0 + 30 * _rndGen.nextDouble()) * 1000; + LOG(error, "Will restart by abort in %d ms.", sleepMS); + std::this_thread::sleep_for(std::chrono::milliseconds(sleepMS)); + _needRestart.store(true); + } } } @@ -298,16 +313,13 @@ Fdispatch::Init(void) _timeouts = 0; _checkLimit = 60; - FS4PersistentPacketStreamer::Instance.SetCompressionLimit( - _config->packetcompresslimit); - FS4PersistentPacketStreamer::Instance.SetCompressionLevel( - _config->packetcompresslevel); - FS4PersistentPacketStreamer::Instance.SetCompressionType( - convert(_config->packetcompresstype)); + FS4PersistentPacketStreamer::Instance.SetCompressionLimit(_config->packetcompresslimit); + FS4PersistentPacketStreamer::Instance.SetCompressionLevel(_config->packetcompresslevel); + FS4PersistentPacketStreamer::Instance.SetCompressionType(convert(_config->packetcompresstype)); LOG(debug, "Creating FNET transport"); - _transport = new FNET_Transport(_config->transportthreads); + _transport = std::make_unique<FNET_Transport>(_config->transportthreads); // grab node slowness limit defaults @@ -317,22 +329,20 @@ Fdispatch::Init(void) FastS_DataSetDesc::SetDefaultSlowDocsumLimitBias(_config->defaultslowdocsumlimitbias); maxthreads = _config->maxthreads; - _mypool = new FastOS_ThreadPool(256 * 1024, maxthreads); + _mypool = std::make_unique<FastOS_ThreadPool>(256 * 1024, maxthreads); // Max interval betw read from socket. FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] = _config->maxsocksilent; - if (_transport != NULL) + if (_transport) { _transport->SetIOCTimeOut((uint32_t) (FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] * 1000.0)); + } char timestr[40]; - FastS_TimeOut::WriteTime(timestr, sizeof(timestr), - FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]); - LOG(debug, - "VERBOSE: Max time between successful read from a socket: %s", - timestr); + FastS_TimeOut::WriteTime(timestr, sizeof(timestr), FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]); + LOG(debug, "VERBOSE: Max time between successful read from a socket: %s", timestr); FastS_QueryCacheUtil::_systemMaxHits = std::numeric_limits<int>::max(); LOG(debug, "VERBOSE: maxhits: %d", FastS_QueryCacheUtil::_systemMaxHits); @@ -341,9 +351,7 @@ Fdispatch::Init(void) const uint32_t linesize = 1; if (FastS_QueryCacheUtil::_systemMaxHits < linesize && FastS_QueryCacheUtil::_maxOffset < linesize - FastS_QueryCacheUtil::_systemMaxHits) { - LOG(warning, - "maxoffset must be >= %d! (overriding config value)", - linesize - FastS_QueryCacheUtil::_systemMaxHits); + LOG(warning, "maxoffset must be >= %d! (overriding config value)", linesize - FastS_QueryCacheUtil::_systemMaxHits); FastS_QueryCacheUtil::_maxOffset = linesize - FastS_QueryCacheUtil::_systemMaxHits; } LOG(debug, "VERBOSE: maxoffset: %d", FastS_QueryCacheUtil::_maxOffset); @@ -354,25 +362,21 @@ Fdispatch::Init(void) LOG(debug, "Using port number %d", ptportnum); - _nodeManager = new FastS_NodeManager(_componentConfig, this, _partition); - + _nodeManager = std::make_unique<FastS_NodeManager>(_componentConfig, this, _partition); GetFNETTransport()->SetTCPNoDelay(_config->transportnodelay); GetFNETTransport()->SetDirectWrite(_config->transportdirectwrite); assert (ptportnum != 0); - _engineAdapter = new fdispatch::EngineAdapter(this, _mypool); - _transportServer = new search::engine::TransportServer - (*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL); + _engineAdapter = std::make_unique<fdispatch::EngineAdapter>(this, _mypool.get()); + _transportServer = std::make_unique<TransportServer>(*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL); _transportServer->setTCPNoDelay(_config->transportnodelay); _transportServer->setDirectWrite(_config->transportdirectwrite); if (!_transportServer->start()) { - delete _transportServer; - _transportServer = NULL; - delete _engineAdapter; - _engineAdapter = NULL; + _transportServer.reset(); + _engineAdapter.reset(); LOG(error, "CRITICAL: Failed to init upwards FNET transport on port %d", ptportnum); return false; } @@ -380,21 +384,19 @@ Fdispatch::Init(void) _nodeManager->SubscribePartMap(_configUri); if (_config->frtport != 0) { - _rpc = new FastS_fdispatch_RPC(this); - FastS_assert(_rpc != NULL); + _rpc = std::make_unique<FastS_fdispatch_RPC>(this); if (!_rpc->Init(_config->frtport, _configUri.getConfigId())) { LOG(error, "RPC init failed"); - delete _rpc; - _rpc = NULL; + _rpc.reset(); } } else { - _rpc = NULL; + _rpc.reset(); } // Kick off fdispatch administrative threads. - if (_transport != NULL) { + if (_transport) { _FNET_adapter.init(); - bool rc = _transport->Start(_mypool); + bool rc = _transport->Start(_mypool.get()); if (rc) { LOG(debug, "Started FNET transport"); _transportStarted = true; @@ -403,7 +405,7 @@ Fdispatch::Init(void) } } FastOS_Thread::Sleep(1000); - if (_rpc != NULL) { + if (_rpc) { _rpc->Start(); } _healthPort = _config->healthport; diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h index b4ff8d9f37e..f4fa80a77ab 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h @@ -10,6 +10,7 @@ #include <vespa/searchcore/config/config-fdispatchrc.h> #include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/util/random.h> class FastS_NodeManager; class FastS_fdispatch_RPC; @@ -53,33 +54,40 @@ public: /** * Note: There is only one instance of this. */ -class Fdispatch : public FastS_AppContext +class Fdispatch : public FastS_AppContext, + public config::IFetcherCallback<vespa::config::search::core::FdispatchrcConfig> { private: + typedef search::engine::TransportServer TransportServer; + typedef vespa::config::search::core::FdispatchrcConfig FdispatchrcConfig; Fdispatch(const Fdispatch &); Fdispatch& operator=(const Fdispatch &); - FastOS_ThreadPool *_mypool; - EngineAdapter *_engineAdapter; - search::engine::TransportServer *_transportServer; + std::unique_ptr<FastOS_ThreadPool> _mypool; + std::unique_ptr<EngineAdapter> _engineAdapter; + std::unique_ptr<TransportServer> _transportServer; vespalib::SimpleComponentConfigProducer _componentConfig; - FastS_NodeManager *_nodeManager; - FNET_Transport *_transport; - FastS_FNETAdapter _FNET_adapter; - FastS_fdispatch_RPC *_rpc; - std::unique_ptr<vespa::config::search::core::FdispatchrcConfig> _config; - config::ConfigUri _configUri; + std::unique_ptr<FastS_NodeManager> _nodeManager; + std::unique_ptr<FNET_Transport> _transport; + FastS_FNETAdapter _FNET_adapter; + std::unique_ptr<FastS_fdispatch_RPC> _rpc; + std::unique_ptr<FdispatchrcConfig> _config; + config::ConfigUri _configUri; + config::ConfigFetcher _fdispatchrcFetcher; + vespalib::RandomGen _rndGen; unsigned int _partition; - bool _tempFail; - bool _FNETLiveCounterDanger; - bool _FNETLiveCounterWarned; - bool _FNETLiveCounterFailed; - bool _transportStarted; + bool _tempFail; + bool _FNETLiveCounterDanger; + bool _FNETLiveCounterWarned; + bool _FNETLiveCounterFailed; + bool _transportStarted; unsigned int _lastFNETLiveCounter; - FastOS_Time _FNETLiveCounterDangerStart; + FastOS_Time _FNETLiveCounterDangerStart; unsigned int _timeouts; unsigned int _checkLimit; - int _healthPort; + int _healthPort; + std::atomic<bool> _needRestart; + void configure(std::unique_ptr<FdispatchrcConfig> cfg); public: // Implements FastS_AppContext virtual FNET_Transport *GetFNETTransport(); @@ -95,10 +103,6 @@ public: int getHealthPort() const { return _healthPort; } vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; } - void - CheckCacheMaxEntries(unsigned int queryCacheMaxEntries, - unsigned int queryAttrCacheMaxEntries); - Fdispatch(const config::ConfigUri &configUri); ~Fdispatch(void); }; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 7b66a4b0105..ea26ef20788 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -178,9 +178,8 @@ DocumentMetaStore::insert(DocId lid, metaData.getBucketId().stripUnused(), metaData.getTimestamp(), _subDbType); - if (state.isActive()) { - _lidAlloc.markAsActive(lid); - } + _lidAlloc.updateActiveLids(lid, state.isActive()); + _lidAlloc.commitActiveLids(); updateCommittedDocIdLimit(); return true; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 2e22b14eccc..244fb940f13 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -90,16 +90,6 @@ LidAllocator::ensureSpace(DocId lid, } void -LidAllocator::markAsActive(DocId lid) -{ - if (_activeLids.get(lid) == 0) { - ++_numActiveLids; - } - _activeLids.update(lid, 1); - _activeLids.commit(); -} - -void LidAllocator::unregisterLid(DocId lid) { assert(!_pendingHoldLids.testBit(lid)); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 0922f9d4edd..836d4c12cbb 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -45,7 +45,6 @@ public: uint32_t newSize, uint32_t newCapacity); void registerLid(DocId lid) { _usedLids.setBit(lid); } - void markAsActive(DocId lid); void unregisterLid(DocId lid); size_t getUsedLidsSize() const; void trimHoldLists(generation_t firstUsed); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp index 7f436a8d594..1624bc3b51c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp @@ -10,13 +10,11 @@ namespace proton { FlushContext::FlushContext( const IFlushHandler::SP &handler, const IFlushTarget::SP &target, - search::SerialNum oldestFlushable, search::SerialNum lastSerial) : _name(createName(*handler, *target)), _handler(handler), _target(target), _task(), - _oldestFlushable(oldestFlushable), _lastSerial(lastSerial) { // empty diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h index 9f2b557b3a9..837a99f153e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.h @@ -19,7 +19,6 @@ private: IFlushHandler::SP _handler; IFlushTarget::SP _target; searchcorespi::FlushTask::UP _task; - search::SerialNum _oldestFlushable; search::SerialNum _lastSerial; public: @@ -43,7 +42,6 @@ public: */ FlushContext(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, - search::SerialNum oldestFlushable, search::SerialNum lastSerial); /** @@ -83,13 +81,6 @@ public: const IFlushTarget::SP & getTarget() const { return _target; } /** - * Returns the oldest flushable serial number. - * - * @return The oldest flushable serial number - */ - search::SerialNum getOldestFlushable() const { return _oldestFlushable; } - - /** * Returns the last serial number. * * @return The last serial number diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index b1a2e23fca0..9d5cf4c006f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -22,16 +22,13 @@ namespace { search::SerialNum findOldestFlushedSerial(const IFlushTarget::List &lst, - const IFlushHandler &handler, - const IFlushTarget *self) + const IFlushHandler &handler) { search::SerialNum ret(handler.getCurrentSerialNumber()); for (const IFlushTarget::SP & target : lst) { - if (self != target.get()) { - ret = std::min(ret, target->getFlushedSerialNum()); - } + ret = std::min(ret, target->getFlushedSerialNum()); } - LOG(debug, "Oldest flushed serial for '%s' will be %" PRIu64 " after flush.", handler.getName().c_str(), ret); + LOG(debug, "Oldest flushed serial for '%s' is %" PRIu64 ".", handler.getName().c_str(), ret); return ret; } @@ -55,11 +52,10 @@ FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, IFlushStrategy::SP strategy, uint32_t numThreads, - uint32_t idleIntervalMS, bool enableAutoPrune) + uint32_t idleIntervalMS) : _closed(false), _maxConcurrent(numThreads), _idleIntervalMS(idleIntervalMS), - _enableAutoPrune(enableAutoPrune), _taskId(0), _threadPool(128 * 1024), _strategy(strategy), @@ -70,7 +66,8 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> _flushing(), _strategyLock(), _strategyMonitor(), - _tlsStatsFactory(tlsStatsFactory) + _tlsStatsFactory(tlsStatsFactory), + _pendingPrune() { // empty } @@ -93,8 +90,8 @@ FlushEngine::start() FlushEngine & FlushEngine::close() { - MonitorGuard strategyGuard(_strategyMonitor); { + MonitorGuard strategyGuard(_strategyMonitor); MonitorGuard guard(_monitor); _closed = true; guard.broadcast(); @@ -128,10 +125,10 @@ bool FlushEngine::wait(size_t minimumWaitTimeIfReady) { MonitorGuard guard(_monitor); - if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard)) { + if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard) && _pendingPrune.empty()) { guard.wait(minimumWaitTimeIfReady); } - while ( ! canFlushMore(guard) ) { + while ( ! canFlushMore(guard) && _pendingPrune.empty()) { guard.wait(1000); // broadcast when flush done } return !_closed; @@ -146,6 +143,9 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) vespalib::string prevFlushName; while (wait(shouldIdle ? _idleIntervalMS : 0)) { shouldIdle = false; + if (prune()) { + continue; // Prune attempted on one or more handlers + } prevFlushName = flushNextTarget(prevFlushName); if ( ! prevFlushName.empty()) { // Sleep at least 10 ms after a successful flush in order to avoid busy loop in case @@ -154,23 +154,26 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg) } else { shouldIdle = true; } - if (_enableAutoPrune) { - prune(); - } LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str()); } } -void FlushEngine::prune() +bool +FlushEngine::prune() { - if (_flushing.empty()) { + std::set<IFlushHandler::SP> toPrune; + { MonitorGuard guard(_monitor); - for (const auto & it : _handlers) { - IFlushHandler & handler(*it.second); - IFlushTarget::List lst = handler.getFlushTargets(); - handler.flushDone(findOldestFlushedSerial(lst, handler, NULL)); + if (_pendingPrune.empty()) { + return false; } + _pendingPrune.swap(toPrune); } + for (const auto &handler : toPrune) { + IFlushTarget::List lst = handler->getFlushTargets(); + handler->flushDone(findOldestFlushedSerial(lst, *handler)); + } + return true; } bool FlushEngine::isFlushing(const MonitorGuard & guard, const vespalib::string & name) const @@ -201,7 +204,6 @@ FlushEngine::getTargetList(bool includeFlushingTargets) const if (!isFlushing(guard, FlushContext::createName(handler, *target)) || includeFlushingTargets) { ret.push_back(FlushContext::SP(new FlushContext(it.second, IFlushTarget::SP(new CachedFlushTarget(target)), - findOldestFlushedSerial(lst, handler, target.get()), serial))); } else { LOG(debug, "Target '%s' with flushedSerialNum = %ld already has a flush going. Local last serial = %ld.", @@ -263,7 +265,7 @@ FlushEngine::flushAll(const FlushContext::List &lst) ctx->getName().c_str(), ctx->getTarget()->getFlushedSerialNum() + 1, ctx->getHandler()->getCurrentSerialNumber()); - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable()))); + _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); } else { LOG(debug, "Target '%s' failed to initiate flush of transactions %" PRIu64 " through %" PRIu64 ".", ctx->getName().c_str(), @@ -303,7 +305,7 @@ FlushEngine::flushNextTarget(const vespalib::string & name) name.c_str(), lst.first.size()); FastOS_Thread::Sleep(1000); } - _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx, ctx->getOldestFlushable()))); + _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx))); return ctx->getName(); } @@ -340,6 +342,8 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, duration.sec()); MonitorGuard guard(_monitor); _flushing.erase(taskId); + assert(ctx.getHandler()); + _pendingPrune.insert(ctx.getHandler()); guard.broadcast(); } @@ -348,7 +352,12 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler) { MonitorGuard guard(_monitor); - return _handlers.putHandler(docTypeName, flushHandler); + IFlushHandler::SP result(_handlers.putHandler(docTypeName, flushHandler)); + if (result) { + _pendingPrune.erase(result); + } + _pendingPrune.insert(flushHandler); + return std::move(result); } IFlushHandler::SP @@ -362,7 +371,9 @@ IFlushHandler::SP FlushEngine::removeFlushHandler(const DocTypeName &docTypeName) { MonitorGuard guard(_monitor); - return _handlers.removeHandler(docTypeName); + IFlushHandler::SP result(_handlers.removeHandler(docTypeName)); + _pendingPrune.erase(result); + return std::move(result); } FlushEngine::FlushMetaSet @@ -395,6 +406,7 @@ FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP void FlushEngine::setStrategy(IFlushStrategy::SP strategy) { + vespalib::LockGuard strategyLock(_strategyLock); MonitorGuard strategyGuard(_strategyMonitor); if (_closed) { return; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 215a8ac8de5..d8d77c11b8f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -49,7 +49,6 @@ private: bool _closed; const uint32_t _maxConcurrent; const uint32_t _idleIntervalMS; - const bool _enableAutoPrune; uint32_t _taskId; FastOS_ThreadPool _threadPool; IFlushStrategy::SP _strategy; @@ -61,13 +60,14 @@ private: vespalib::Lock _strategyLock; // serialize setStrategy calls vespalib::Monitor _strategyMonitor; std::shared_ptr<flushengine::ITlsStatsFactory> _tlsStatsFactory; + std::set<IFlushHandler::SP> _pendingPrune; FlushContext::List getTargetList(bool includeFlushingTargets) const; std::pair<FlushContext::List,bool> getSortedTargetList(vespalib::MonitorGuard &strategyGuard) const; FlushContext::SP initNextFlush(const FlushContext::List &lst); vespalib::string flushNextTarget(const vespalib::string & name); void flushAll(const FlushContext::List &lst); - void prune(); + bool prune(); uint32_t initFlush(const FlushContext &ctx); uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); void flushDone(const FlushContext &ctx, uint32_t taskId); @@ -93,15 +93,10 @@ public: * @param strategy The flushing strategy to use. * @param numThreads The number of worker threads to use. * @param idleInterval The interval between when flushes are checked whne there are no one progressing. - * @param enableAutoPrune Indicate if pruning shall be done even if there - are no flushing happening. Turn off for some tests. - Needed for pruning to be correct if one flush is started - while another is in progress. In that case the pruning - will be too conservative. */ FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, - IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS, bool enableAutoPrune); + IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS); /** * Destructor. Waits for all pending tasks to complete. diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp index fc692979753..8af59d3079f 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp @@ -9,12 +9,10 @@ namespace proton { FlushTask::FlushTask(uint32_t taskId, FlushEngine &engine, - const FlushContext::SP &ctx, - search::SerialNum serial) + const FlushContext::SP &ctx) : _taskId(taskId), _engine(engine), - _context(ctx), - _serial(serial) + _context(ctx) { LOG_ASSERT(_context.get() != NULL); } @@ -34,7 +32,6 @@ FlushTask::run() } task->run(); task.reset(); - _context->getHandler()->flushDone(_serial); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h index fd27538ca4b..c10fb740410 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h @@ -25,13 +25,10 @@ public: * @param taskId The identifier used by IFlushStrategy. * @param engine The running flush engine. * @param ctx The context of the flush to perform. - * @param serial The oldest unflushed serial available in the handler once - * this task has been run. */ FlushTask(uint32_t taskId, FlushEngine &engine, - const FlushContext::SP &ctx, - search::SerialNum serial); + const FlushContext::SP &ctx); /** * Destructor. Notifies the engine that the flush is done to prevent the diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index 6912cbfde13..2d3eddf7af9 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -76,9 +76,10 @@ public: * up to the given serial number can be pruned from the domain of this * handler. This method is called by an arbitrary worker thread. * - * @param oldestSerial The oldest transaction that is still in use. + * @param flushedSerial Serial number flushed for all flush + * targets belonging to this handler. */ - virtual void flushDone(SerialNum oldestSerial) = 0; + virtual void flushDone(SerialNum flushedSerial) = 0; /* * This method is called to sync tls to stable media, up to and diff --git a/searchcore/src/vespa/searchcore/proton/index/OWNERS b/searchcore/src/vespa/searchcore/proton/index/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchcore/src/vespa/searchcore/proton/index/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/index/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp index 2acb9c3a165..8667cdb234b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp @@ -24,7 +24,6 @@ using search::fef::FieldType; using search::fef::IIndexEnvironment; using search::fef::MatchData; using search::fef::MatchDataLayout; -using search::fef::SimpleTermData; using search::fef::TermFieldHandle; using search::query::Node; using search::query::TemplateTermVisitor; @@ -109,25 +108,12 @@ ProtonTermData::setDocumentFrequency(uint32_t estHits, uint32_t docIdLimit) } } -size_t -ProtonTermData::numFields() const -{ - return _fields.size(); -} - -const ProtonTermData::FieldEntry & -ProtonTermData::field(size_t i) const -{ - assert(i < _fields.size()); - return _fields[i]; -} - const ProtonTermData::FieldEntry * ProtonTermData::lookupField(uint32_t fieldId) const { - for (size_t i = 0; i < numFields(); ++i) { - if (field(i).getFieldId() == fieldId) { - return &field(i); + for (size_t i = 0; i < _fields.size(); ++i) { + if (_fields[i].getFieldId() == fieldId) { + return &_fields[i]; } } return 0; diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h index 63052d633d1..b0f844b779c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h @@ -27,7 +27,7 @@ class ProtonTermData : public search::fef::ITermData public: typedef search::queryeval::FieldSpec FieldSpec; - struct FieldEntry : search::fef::SimpleTermFieldData { + struct FieldEntry final : search::fef::SimpleTermFieldData { vespalib::string field_name; bool attribute_field; bool filter_field; @@ -42,7 +42,7 @@ public: return FieldSpec(field_name, getFieldId(), getHandle(), filter_field); } - virtual search::fef::TermFieldHandle getHandle() const; + search::fef::TermFieldHandle getHandle() const override; }; private: @@ -62,9 +62,9 @@ public: void setDocumentFrequency(uint32_t estHits, uint32_t numDocs); // ITermData interface - virtual size_t numFields() const; - virtual const FieldEntry &field(size_t i) const; - virtual const FieldEntry *lookupField(uint32_t fieldId) const; + size_t numFields() const override final { return _fields.size(); } + const FieldEntry &field(size_t i) const override final { return _fields[i]; } + const FieldEntry *lookupField(uint32_t fieldId) const override final; }; namespace { @@ -76,8 +76,8 @@ uint32_t numTerms<search::query::Phrase>(const search::query::Phrase &n) { } // namespace proton::matching::<unnamed> template <typename Base> -struct ProtonTerm : public Base, - public ProtonTermData +struct ProtonTermBase : public Base, + public ProtonTermData { using Base::Base; @@ -89,10 +89,15 @@ struct ProtonTerm : public Base, } // ITermData interface - virtual uint32_t getPhraseLength() const { return numTerms<Base>(*this); } - virtual uint32_t getTermIndex() const { return -1; } - virtual search::query::Weight getWeight() const { return Base::getWeight(); } - virtual uint32_t getUniqueId() const { return Base::getId(); } + uint32_t getPhraseLength() const override final { return numTerms<Base>(*this); } + uint32_t getTermIndex() const override final { return -1; } + search::query::Weight getWeight() const override final { return Base::getWeight(); } + uint32_t getUniqueId() const override final { return Base::getId(); } +}; + +template <typename Base> +struct ProtonTerm : public ProtonTermBase<Base> { + using ProtonTermBase<Base>::ProtonTermBase; }; typedef search::query::SimpleAnd ProtonAnd; @@ -103,10 +108,10 @@ typedef search::query::SimpleOr ProtonOr; typedef search::query::SimpleRank ProtonRank; typedef search::query::SimpleWeakAnd ProtonWeakAnd; -struct ProtonEquiv : public ProtonTerm<search::query::Equiv> +struct ProtonEquiv final : public ProtonTermBase<search::query::Equiv> { search::fef::MatchDataLayout children_mdl; - using ProtonTerm::ProtonTerm; + using ProtonTermBase::ProtonTermBase; }; typedef ProtonTerm<search::query::LocationTerm> ProtonLocationTerm; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp index 606312fcae2..06a20e8da78 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp @@ -12,14 +12,16 @@ TransLogServerMetrics::DomainMetrics::DomainMetrics(metrics::MetricSet *parent, const vespalib::string &documentType) : metrics::MetricSet("transactionlog", {{"documenttype", documentType}}, "Transaction log metrics for a document type", parent), - entries("entries", "", "The current number of entries in the transaction log", this) + entries("entries", "", "The current number of entries in the transaction log", this), + diskUsage("disk_usage", "", "The disk usage (in bytes) of the transaction log", this) { } void TransLogServerMetrics::DomainMetrics::update(const DomainInfo &stats) { - entries.set(stats.count); + entries.set(stats.numEntries); + diskUsage.set(stats.byteSize); } void diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h index 0d52f6cf0d1..4a7241bbef7 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.h @@ -16,6 +16,7 @@ public: struct DomainMetrics : public metrics::MetricSet { metrics::LongValueMetric entries; + metrics::LongValueMetric diskUsage; typedef std::unique_ptr<DomainMetrics> UP; DomainMetrics(metrics::MetricSet *parent, const vespalib::string &documentType); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 63f3850583c..759fd245670 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -125,7 +125,8 @@ public: Match(const IDocumentRetriever & source, bool metaOnly, const vespalib::string & selection) : _dscTrue(true), _metaOnly(metaOnly), - _willAlwaysFail(false) + _willAlwaysFail(false), + _docidLimit(source.getDocIdLimit()) { if (!(_metaOnly || selection.empty())) { LOG(spam, "ParseSelect: %s", selection.c_str()); @@ -158,6 +159,9 @@ public: bool willAlwaysFail() const { return _willAlwaysFail; } bool match(const search::DocumentMetaData & meta) const { + if (meta.lid >= _docidLimit) { + return false; + } if (_dscTrue || _metaOnly) { return true; } @@ -181,9 +185,10 @@ private: bool _dscTrue; bool _metaOnly; bool _willAlwaysFail; + uint32_t _docidLimit; CachedSelect::SP _cs; document::select::Node::UP _select; - document::select::GidFilter _gidFilter; + document::select::GidFilter _gidFilter; std::unique_ptr<SelectContext> _sc; }; @@ -226,6 +231,7 @@ private: void DocumentIterator::fetchCompleteSource(const IDocumentRetriever & source, IterateResult::List & list) { + IDocumentRetriever::ReadGuard sourceReadGuard(source.getReadGuard()); search::DocumentMetaData::Vector metaData; source.getBucketMetaData(_bucket, metaData); if (metaData.empty()) { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h index cf0f8e8334c..5d66e2e52a1 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/common/cachedselect.h> #include <vespa/searchlib/query/base.h> #include <memory> +#include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h> namespace proton { @@ -24,6 +25,7 @@ class IDocumentRetriever { public: using ReadConsistency = storage::spi::ReadConsistency; + using ReadGuard = IDocumentMetaStoreContext::IReadGuard::UP; typedef std::unique_ptr<IDocumentRetriever> UP; typedef std::shared_ptr<IDocumentRetriever> SP; @@ -34,6 +36,8 @@ public: virtual void getBucketMetaData(const storage::spi::Bucket &bucket, search::DocumentMetaData::Vector &result) const = 0; virtual search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const = 0; virtual document::Document::UP getDocument(search::DocumentIdT lid) const = 0; + virtual ReadGuard getReadGuard() const = 0; + virtual uint32_t getDocIdLimit() const = 0; /** * Will visit all documents in the the given list. Visit order is undefined and will * be conducted in most efficient retrieval order. @@ -47,7 +51,9 @@ public: class DocumentRetrieverBaseForTest : public IDocumentRetriever { public: - virtual void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const override; + void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const override; + ReadGuard getReadGuard() const override { return ReadGuard(); } + uint32_t getDocIdLimit() const override { return std::numeric_limits<uint32_t>::max(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h index 3e558b71e30..38e7f85fda4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h @@ -85,7 +85,7 @@ public: const storage::spi::Bucket &target1, const storage::spi::Bucket &target2) = 0; - virtual RetrieversSP getDocumentRetrievers() = 0; + virtual RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency consistency) = 0; virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &bucket) = 0; virtual void diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index a81adf1134a..fd02fbb126c 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -440,14 +440,14 @@ GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, - Context&) const + Context& context) const { RWLockReader rguard(getRLock()); HandlerSnapshot::UP snapshot = getHandlerSnapshot(); for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b); - IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(); + IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { IDocumentRetriever &retriever = *(*retrievers)[i]; search::DocumentMetaData meta = retriever.getDocumentMetaData(did); @@ -478,12 +478,12 @@ PersistenceEngine::createIterator(const Bucket &bucket, RWLockReader rguard(getRLock()); HandlerSnapshot::UP snapshot = getHandlerSnapshot(); - IteratorEntry *entry = new IteratorEntry(context.getReadConsistency(), bucket, fields, selection, - versions, _defaultSerializedSize, _ignoreMaxBytes); + auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection, + versions, _defaultSerializedSize, _ignoreMaxBytes); entry->bucket_guards.reserve(snapshot->size()); for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket)); - IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(); + IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { entry->it.add((*retrievers)[i]); } @@ -493,7 +493,7 @@ PersistenceEngine::createIterator(const Bucket &bucket, LockGuard guard(_iterators_lock); static IteratorId id_counter(0); IteratorId id(++id_counter); - _iterators[id] = entry; + _iterators[id] = entry.release(); return CreateIteratorResult(id); } diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/server/OWNERS b/searchcore/src/vespa/searchcore/proton/server/OWNERS index 7c03446a5d4..614d678c95e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/OWNERS +++ b/searchcore/src/vespa/searchcore/proton/server/OWNERS @@ -1,2 +1,2 @@ geirst -tegge +toregge diff --git a/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h index 3735240d55d..f75463b754d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/commit_and_wait_document_retriever.h @@ -44,15 +44,19 @@ public: void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const override { - if (readConsistency == ReadConsistency::STRONG) { - _commit.commitAndWait(); - } + _commit.commitAndWait(); _retriever->visitDocuments(lids, visitor, readConsistency); } CachedSelect::SP parseSelect(const vespalib::string &selection) const override { return _retriever->parseSelect(selection); } + ReadGuard getReadGuard() const override { + return _retriever->getReadGuard(); + } + uint32_t getDocIdLimit() const override { + return _retriever->getDocIdLimit(); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h index ee6352d61d5..38d0dcfe7bd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/configstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h @@ -33,6 +33,12 @@ struct ConfigStore : FeedConfigStore { SerialNum serialNum) = 0; virtual void removeInvalid() = 0; + /** + * Perform prune after everything up to and including serialNum has been + * flushed to stable storage. + * + * @param serialNum The serial number flushed to stable storage. + */ virtual void prune(SerialNum serialNum) = 0; virtual SerialNum getBestSerialNum() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index eaa166afe9c..f2d54d8fbf2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -40,6 +40,7 @@ LOG_SETUP(".proton.server.documentdb"); #include <vespa/vespalib/util/jsonwriter.h> #include <sstream> #include "documentdbconfigscout.h" +#include "commit_and_wait_document_retriever.h" using vespa::config::search::AttributesConfig; @@ -648,13 +649,13 @@ DocumentDB::onTransactionLogReplayDone() void -DocumentDB::onPerformPrune(SerialNum oldestSerial) +DocumentDB::onPerformPrune(SerialNum flushedSerial) { if (!getAllowPrune()) { assert(_state.getClosed()); return; } - _config_store->prune(oldestSerial); + _config_store->prune(flushedSerial); } @@ -716,12 +717,26 @@ DocumentDB::startTransactionLogReplay() BucketGuard::UP DocumentDB::lockBucket(const document::BucketId &bucket) { - BucketGuard::UP guard(std::make_unique<BucketGuard> - (bucket, _maintenanceController)); - _visibility.commitAndWait(); + BucketGuard::UP guard(std::make_unique<BucketGuard>(bucket, _maintenanceController)); return std::move(guard); } +std::shared_ptr<std::vector<IDocumentRetriever::SP> > +DocumentDB::getDocumentRetrievers(IDocumentRetriever::ReadConsistency consistency) +{ + std::shared_ptr<std::vector<IDocumentRetriever::SP> > list = _subDBs.getRetrievers(); + + if (consistency == IDocumentRetriever::ReadConsistency::STRONG) { + std::shared_ptr<std::vector<IDocumentRetriever::SP> > wrappedList = std::make_shared<std::vector<IDocumentRetriever::SP>>(); + wrappedList->reserve(list->size()); + for (const IDocumentRetriever::SP & retriever : *list) { + wrappedList->emplace_back(new CommitAndWaitDocumentRetriever(retriever, _visibility)); + } + return wrappedList; + } else { + return list; + } +} SerialNum DocumentDB::getOldestFlushedSerial() @@ -760,9 +775,9 @@ DocumentDB::getFlushTargets() } void -DocumentDB::flushDone(SerialNum oldestSerial) +DocumentDB::flushDone(SerialNum flushedSerial) { - _feedHandler.flushDone(oldestSerial); + _feedHandler.flushDone(flushedSerial); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 0c49c2c0170..cabb4434591 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -201,7 +201,7 @@ private: * Implements FeedHandler::IOwner */ virtual void onTransactionLogReplayDone() __attribute__((noinline)); - virtual void onPerformPrune(SerialNum oldestSerial); + virtual void onPerformPrune(SerialNum flushedSerial); virtual bool isFeedBlockedByRejectedConfig(); /** @@ -370,7 +370,7 @@ public: * reconfiguration, however. */ std::shared_ptr<std::vector<IDocumentRetriever::SP> > - getDocumentRetrievers() { return _subDBs.getRetrievers(); } + getDocumentRetrievers(IDocumentRetriever::ReadConsistency consistency); MaintenanceController &getMaintenanceController() { return _maintenanceController; @@ -391,7 +391,7 @@ public: getDocsums(const search::engine::DocsumRequest & request); IFlushTarget::List getFlushTargets(); - void flushDone(SerialNum oldestSerial); + void flushDone(SerialNum flushedSerial); virtual SerialNum getCurrentSerialNumber() const diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h index 4e94461e5bc..4894baf46e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h @@ -12,8 +12,6 @@ namespace proton { -class IDocumentMetaStoreContext; - class DocumentRetrieverBase : public IDocumentRetriever { const DocTypeName &_docTypeName; @@ -41,6 +39,8 @@ public: search::DocumentMetaData::Vector &result) const override; search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override; CachedSelect::SP parseSelect(const vespalib::string &selection) const override; + ReadGuard getReadGuard() const override { return _meta_store.getReadGuard(); } + uint32_t getDocIdLimit() const override { return _meta_store.getReadGuard()->get().getCommittedDocIdLimit(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index cb8abd1cfd9..a95794742e2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -295,7 +295,7 @@ FastAccessDocSubDB::getDocumentRetriever() { FastAccessFeedView::SP feedView = _fastUpdateFeedView.get(); proton::IAttributeManager::SP attrMgr = extractAttributeManager(feedView); - return IDocumentRetriever::UP(new FastAccessDocumentRetriever(feedView, attrMgr)); + return IDocumentRetriever::UP(new FastAccessDocumentRetriever(feedView, attrMgr, _docIdLimit)); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h index 907939a5a50..7ff43e20b0d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_document_retriever.h @@ -19,10 +19,12 @@ class FastAccessDocumentRetriever : public DocumentRetriever private: FastAccessFeedView::SP _feedView; IAttributeManager::SP _attrMgr; + const DocIdLimit &_docIdLimit; public: FastAccessDocumentRetriever(const FastAccessFeedView::SP &feedView, - const IAttributeManager::SP &attrMgr) + const IAttributeManager::SP &attrMgr, + const DocIdLimit & docIdLimit) : DocumentRetriever(feedView->getPersistentParams()._docTypeName, *feedView->getDocumentTypeRepo(), *feedView->getSchema(), @@ -30,9 +32,11 @@ public: *attrMgr, feedView->getDocumentStore()), _feedView(feedView), - _attrMgr(attrMgr) + _attrMgr(attrMgr), + _docIdLimit(docIdLimit) { } + uint32_t getDocIdLimit() const override { return _docIdLimit.get(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 61e2a83bd37..6b7eadb2cb8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -432,32 +432,32 @@ FeedHandler::performEof() void -FeedHandler::performFlushDone(SerialNum oldestSerial) +FeedHandler::performFlushDone(SerialNum flushedSerial) { assert(_writeService.master().isCurrentThread()); - // XXX: oldestSerial can go backwards when attribute vectors are + // XXX: flushedSerial can go backwards when attribute vectors are // resurrected. This can be avoided if resurrected attribute vectors // pretends to have been flushed at resurrect time. - if (oldestSerial <= _prunedSerialNum) { + if (flushedSerial <= _prunedSerialNum) { return; // Cannot unprune. } if (!_owner.getAllowPrune()) { - _prunedSerialNum = oldestSerial; + _prunedSerialNum = flushedSerial; _delayedPrune = true; return; } _delayedPrune = false; - performPrune(oldestSerial); + performPrune(flushedSerial); } void -FeedHandler::performPrune(SerialNum oldestSerial) +FeedHandler::performPrune(SerialNum flushedSerial) { try { - tlsPrune(oldestSerial); // throws on error - LOG(debug, "Pruned TLS to token %" PRIu64 ".", oldestSerial); - _owner.onPerformPrune(oldestSerial); + tlsPrune(flushedSerial); // throws on error + LOG(debug, "Pruned TLS to token %" PRIu64 ".", flushedSerial); + _owner.onPerformPrune(flushedSerial); } catch (const vespalib::IllegalStateException & e) { LOG(warning, "FeedHandler::performPrune failed due to '%s'.", e.what()); } @@ -606,7 +606,7 @@ FeedHandler::replayTransactionLog(SerialNum flushedIndexMgrSerial, void -FeedHandler::flushDone(SerialNum oldestSerial) +FeedHandler::flushDone(SerialNum flushedSerial) { // Called by flush worker thread after performing a flush task _writeService.master().execute( @@ -614,7 +614,7 @@ FeedHandler::flushDone(SerialNum oldestSerial) makeClosure( this, &FeedHandler::performFlushDone, - oldestSerial))); + flushedSerial))); } void FeedHandler::changeToNormalFeedState(void) { diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 536013e0d02..a1bf80181db 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -62,7 +62,7 @@ public: virtual void performWipeHistory() = 0; virtual void onTransactionLogReplayDone() = 0; virtual void enterRedoReprocessState() = 0; - virtual void onPerformPrune(SerialNum oldestSerial) = 0; + virtual void onPerformPrune(SerialNum flushedSerial) = 0; virtual bool isFeedBlockedByRejectedConfig() = 0; virtual bool getAllowPrune() const = 0; }; @@ -153,10 +153,10 @@ private: * Used when flushing is done */ void - performFlushDone(SerialNum oldestSerial); + performFlushDone(SerialNum flushedSerial); void - performPrune(SerialNum oldestSerial); + performPrune(SerialNum flushedSerial); public: void @@ -240,10 +240,10 @@ public: /** * Called when a flush is done and allows pruning of the transaction log. * - * @param oldestSerial The oldest serial number that is still in use. + * @param flushedSerial serial number flushed for all relevant flush targets. */ void - flushDone(SerialNum oldestSerial); + flushDone(SerialNum flushedSerial); /** * Used to flip between normal and recovery feed states. diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp index 7736161ddbc..5599b245655 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.cpp @@ -36,9 +36,9 @@ FlushHandlerProxy::getCurrentSerialNumber(void) const void -FlushHandlerProxy::flushDone(SerialNum oldestSerial) +FlushHandlerProxy::flushDone(SerialNum flushedSerial) { - _documentDB->flushDone(oldestSerial); + _documentDB->flushDone(flushedSerial); } diff --git a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h index 7e4cd1f3176..9f27c29c2bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/flushhandlerproxy.h @@ -28,7 +28,7 @@ public: getCurrentSerialNumber(void) const; virtual void - flushDone(SerialNum oldestSerial); + flushDone(SerialNum flushedSerial); virtual void syncTls(SerialNum syncTo); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index 2f26c1a2601..bbf803adfe4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -83,13 +83,7 @@ MemoryFlush::Config::Config(uint64_t maxGlobalMemory_in, MemoryFlush::MemoryFlush(const Config &config, fastos::TimeStamp startTime) : _lock(), - _globalMaxMemory(config.maxGlobalMemory), - _maxGlobalTlsSize(config.maxGlobalTlsSize), - _globalDiskBloatFactor(config.globalDiskBloatFactor), - _maxMemoryGain(config.maxMemoryGain), - _diskBloatFactor(config.diskBloatFactor), - _maxSerialGain(config.maxSerialGain), - _maxTimeGain(config.maxTimeGain), + _config(config), _startTime(startTime) { } @@ -101,6 +95,18 @@ MemoryFlush::MemoryFlush() // empty } +MemoryFlush::Config MemoryFlush::getConfig() const +{ + vespalib::LockGuard guard(_lock); + return _config; +} + +void MemoryFlush::setConfig(const Config &config) +{ + vespalib::LockGuard guard(_lock); + _config = config; +} + FlushContext::List MemoryFlush::getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap & @@ -110,13 +116,16 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, uint64_t totalMemory(0); IFlushTarget::DiskGain totalDisk; uint64_t totalTlsSize(0); + const Config config(getConfig()); vespalib::hash_set<const void *> visitedHandlers; fastos::TimeStamp now(fastos::ClockSystem::now()); LOG(debug, "getFlushTargets(): globalMaxMemory(%" PRIu64 "), globalDiskBloatFactor(%f), " "maxMemoryGain(%" PRIu64 "), diskBloatFactor(%f), maxSerialGain(%" PRIu64 "), maxTimeGain(%f), startTime(%f)", - _globalMaxMemory, _globalDiskBloatFactor, _maxMemoryGain, _diskBloatFactor, - _maxSerialGain, _maxTimeGain.sec(), _startTime.sec()); + config.maxGlobalMemory, config.globalDiskBloatFactor, + config.maxMemoryGain, config.diskBloatFactor, + config.maxSerialGain, config.maxTimeGain.sec(), + _startTime.sec()); for (size_t i(0), m(targetList.size()); i < m; i++) { const IFlushTarget & target(*targetList[i]->getTarget()); const IFlushHandler & handler(*targetList[i]->getHandler()); @@ -133,26 +142,26 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, tlsStatsMap.getTlsStats(handler.getName()); if (visitedHandlers.insert(&handler).second) { totalTlsSize += tlsStats.getNumBytes(); - if ((totalTlsSize > _maxGlobalTlsSize) && (order < TLSSIZE)) { + if ((totalTlsSize > config.maxGlobalTlsSize) && (order < TLSSIZE)) { order = TLSSIZE; } } - if (((totalMemory >= _globalMaxMemory) || - (mgain >= _maxMemoryGain)) && (order < MEMORY)) { + if (((totalMemory >= config.maxGlobalMemory) || + (mgain >= config.maxMemoryGain)) && (order < MEMORY)) { order = MEMORY; } else if (((totalDisk.gain() > - _globalDiskBloatFactor * std::max(100000000l, + config.globalDiskBloatFactor * std::max(100000000l, std::max(totalDisk.getBefore(), totalDisk.getAfter()))) - || dgain.gain() > _diskBloatFactor * + || dgain.gain() > config.diskBloatFactor * std::max(10000000l, std::max(dgain.getBefore(), dgain.getAfter()))) && (order < DISKBLOAT) ) { order = DISKBLOAT; - } else if ((serialDiff >= _maxSerialGain) && (order < MAXSERIAL)) { + } else if ((serialDiff >= config.maxSerialGain) && (order < MAXSERIAL)) { order = MAXSERIAL; - } else if ((timeDiff >= _maxTimeGain) && (order < MAXAGE)) { + } else if ((timeDiff >= config.maxTimeGain) && (order < MAXAGE)) { order = MAXAGE; } LOG(debug, diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h index cb0c84b6364..165ba4de70e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.h @@ -15,12 +15,21 @@ class MemoryFlush : public boost::noncopyable, public: struct Config { + /// Global maxMemory uint64_t maxGlobalMemory; + /// Maximum global tls size. uint64_t maxGlobalTlsSize; + /// Maximum global disk bloat factor. When this limit is reached + /// flush is forced. double globalDiskBloatFactor; + /// Maximum memory saved. When this limit is reached flush is forced. int64_t maxMemoryGain; + /// Maximum disk bloat factor. When this limit is reached + /// flush is forced. double diskBloatFactor; + /// Maximum count of what a target can have outstanding in the TLS. int64_t maxSerialGain; + /// Maximum age of unflushed data. fastos::TimeStamp maxTimeGain; Config(); Config(uint64_t maxGlobalMemory_in, @@ -35,21 +44,7 @@ public: private: /// Needed as flushDone is called in different context from the rest vespalib::Lock _lock; - /// Global maxMemory - uint64_t _globalMaxMemory; - /// Maximum global tls size. - uint64_t _maxGlobalTlsSize; - /// Maximum global disk bloat factor. When this limit is reached - /// flush is forced. - double _globalDiskBloatFactor; - /// Maximum memory saved. When this limit is reached flush is forced. - int64_t _maxMemoryGain; - /// Maximum disk bloat factor. When this limit is reached flush is forced. - double _diskBloatFactor; - /// Maximum count of what a target can have outstanding in the TLS. - int64_t _maxSerialGain; - /// Maximum age of unflushed data. - fastos::TimeStamp _maxTimeGain; + Config _config; /// The time when the strategy was started. fastos::TimeStamp _startTime; @@ -73,6 +68,8 @@ private: const flushengine::TlsStatsMap &_tlsStatsMap; }; + Config getConfig() const; + public: MemoryFlush(); @@ -84,6 +81,8 @@ public: getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap & tlsStatsMap) const override; + + void setConfig(const Config &config); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp index 706e738d8af..ff811e80f94 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp @@ -167,9 +167,9 @@ PersistenceHandlerProxy::handleJoin(FeedToken token, IPersistenceHandler::RetrieversSP -PersistenceHandlerProxy::getDocumentRetrievers() +PersistenceHandlerProxy::getDocumentRetrievers(storage::spi::ReadConsistency consistency) { - return _documentDB->getDocumentRetrievers(); + return _documentDB->getDocumentRetrievers(consistency); } BucketGuard::UP diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h index 56d16a3144a..3ec92e96439 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.h @@ -73,7 +73,7 @@ public: const storage::spi::Bucket &target1, const storage::spi::Bucket &target2) override; - virtual RetrieversSP getDocumentRetrievers() override; + virtual RetrieversSP getDocumentRetrievers(storage::spi::ReadConsistency consistency) override; virtual BucketGuard::UP lockBucket(const storage::spi::Bucket &bucket) override; virtual void diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 60c54bf1ef3..2750515661f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -89,10 +89,41 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton) proton.writefilter.sampleinterval); } +static constexpr size_t TOTAL_HARD_MEMORY_LIMIT=16*1024*1024*1024ul; +static constexpr size_t EACH_HARD_MEMORY_LIMIT=12*1024*1024*1024ul; + +MemoryFlush::Config +memoryFlushConfig(const ProtonConfig::Flush &flush) +{ + size_t totalMaxMemory = flush.memory.maxmemory; + if (totalMaxMemory > TOTAL_HARD_MEMORY_LIMIT) { + LOG(warning, "flush.memory.maxmemory=%ld can not" + " be set above the hard limit of %ld so we cap it", + flush.memory.maxmemory, + TOTAL_HARD_MEMORY_LIMIT); + totalMaxMemory = TOTAL_HARD_MEMORY_LIMIT; + } + size_t eachMaxMemory = flush.memory.each.maxmemory; + if (eachMaxMemory > EACH_HARD_MEMORY_LIMIT) { + LOG(warning, "flush.memory.each.maxmemory=%ld can not" + " be set above the hard limit of %ld so we cap it", + flush.memory.maxmemory, + EACH_HARD_MEMORY_LIMIT); + eachMaxMemory = EACH_HARD_MEMORY_LIMIT; + } + return MemoryFlush::Config(totalMaxMemory, + flush.memory.maxtlssize, + flush.memory.diskbloatfactor, + eachMaxMemory, + flush.memory.each.diskbloatfactor, + flush.memory.maxage.serial, + static_cast<long> + (flush.memory.maxage.time) * + fastos::TimeStamp::NANO); +} + } -static const size_t TOTAL_HARD_MEMORY_LIMIT=16*1024*1024*1024ul; -static const size_t EACH_HARD_MEMORY_LIMIT=12*1024*1024*1024ul; static const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, @@ -262,34 +293,9 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) IFlushStrategy::SP strategy; const ProtonConfig::Flush & flush(protonConfig.flush); switch (flush.strategy) { - case ProtonConfig::Flush::MEMORY: { - size_t totalMaxMemory = flush.memory.maxmemory; - if (totalMaxMemory > TOTAL_HARD_MEMORY_LIMIT) { - LOG(warning, "flush.memory.maxmemory=%ld can not" - " be set above the hard limit of %ld so we cap it", - flush.memory.maxmemory, - TOTAL_HARD_MEMORY_LIMIT); - totalMaxMemory = TOTAL_HARD_MEMORY_LIMIT; - } - size_t eachMaxMemory = flush.memory.each.maxmemory; - if (eachMaxMemory > EACH_HARD_MEMORY_LIMIT) { - LOG(warning, "flush.memory.each.maxmemory=%ld can not" - " be set above the hard limit of %ld so we cap it", - flush.memory.maxmemory, - EACH_HARD_MEMORY_LIMIT); - eachMaxMemory = EACH_HARD_MEMORY_LIMIT; - } + case ProtonConfig::Flush::MEMORY: strategy = std::make_shared<MemoryFlush>( - MemoryFlush::Config(totalMaxMemory, - flush.memory.maxtlssize, - flush.memory.diskbloatfactor, - eachMaxMemory, - flush.memory.each.diskbloatfactor, - flush.memory.maxage.serial, - static_cast<long> - (flush.memory.maxage.time) * - fastos::TimeStamp::NANO)); - } + memoryFlushConfig(flush)); break; case ProtonConfig::Flush::SIMPLE: default: @@ -299,8 +305,9 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::mkdir(protonConfig.basedir + "/documents", true); vespalib::chdir(protonConfig.basedir); _tls->start(); + _strategy = strategy; _flushEngine.reset(new FlushEngine(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), - strategy, flush.maxconcurrent, flush.idleinterval*1000, true)); + strategy, flush.maxconcurrent, flush.idleinterval*1000)); _fs4Server.reset(new TransportServer(*_matchEngine, *_summaryEngine, *this, protonConfig.ptport, TransportServer::DEBUG_ALL)); _fs4Server->setTCPNoDelay(true); _metricsEngine->addExternalMetrics(_fs4Server->getMetrics()); @@ -538,6 +545,12 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot, configSnapshot->getGeneration())); _diskMemUsageSampler-> setConfig(diskMemUsageSamplerConfig(protonConfig)); + std::shared_ptr<MemoryFlush> memoryFlushStrategy = + std::dynamic_pointer_cast<MemoryFlush>(_strategy); + if (memoryFlushStrategy) { + memoryFlushStrategy->setConfig(memoryFlushConfig(protonConfig.flush)); + _flushEngine->kick(); + } } void diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 023a6173984..be8e602fa2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -106,6 +106,7 @@ private: MatchEngine::UP _matchEngine; SummaryEngine::UP _summaryEngine; DocsumBySlime::UP _docsumBySlime; + IFlushStrategy::SP _strategy; FlushEngine::UP _flushEngine; RPCHooks::UP _rpcHooks; HealthAdapter _healthAdapter; diff --git a/searchcorespi/.gitignore b/searchcorespi/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/searchcorespi/.gitignore +++ b/searchcorespi/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/searchcorespi/OWNERS b/searchcorespi/OWNERS index fda83fc8cf8..23c5db50ab7 100644 --- a/searchcorespi/OWNERS +++ b/searchcorespi/OWNERS @@ -1,3 +1,3 @@ -tegge +toregge geirst baldersheim diff --git a/searchlib/.gitignore b/searchlib/.gitignore index ef6cddb9800..a58e2e56d5e 100644 --- a/searchlib/.gitignore +++ b/searchlib/.gitignore @@ -6,4 +6,3 @@ target *.iws /pom.xml.build Makefile -Testing diff --git a/searchlib/OWNERS b/searchlib/OWNERS index 2b546321fd4..edcc41fe923 100644 --- a/searchlib/OWNERS +++ b/searchlib/OWNERS @@ -1,4 +1,4 @@ havardpe baldersheim -tegge +toregge geirst diff --git a/searchlib/pom.xml b/searchlib/pom.xml index 565370de634..88361289509 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchlib</artifactId> <packaging>container-plugin</packaging> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java index ecbab688821..381e7da994f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java @@ -19,7 +19,6 @@ public class ForceLoad { "VdsHit", "Grouping", "Hit", - "ForceLoad", "MinAggregationResult", "GroupingLevel", "MaxAggregationResult", @@ -27,10 +26,10 @@ public class ForceLoad { "AverageAggregationResult", "ExpressionCountAggregationResult", "hll.SparseSketch", - "hll.NormalSketch", - "ForceLoad" + "hll.NormalSketch" }; - com.yahoo.system.ForceLoad.forceLoad(pkg, classes); + com.yahoo.system.ForceLoad.forceLoad(pkg, classes, + ForceLoad.class.getClassLoader()); } public static boolean forceLoad() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java index 6ebb4c672c8..8b12f4c5967 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java @@ -58,7 +58,6 @@ public class ForceLoad { "FloatResultNodeVector", "StringResultNodeVector", "RawResultNodeVector", - "ForceLoad", "MultiplyFunctionNode", "IntegerBucketResultNode", "FloatBucketResultNode", @@ -77,10 +76,10 @@ public class ForceLoad { "ZCurveFunctionNode", "XorBitFunctionNode", "MultiArgFunctionNode", - "DebugWaitFunctionNode", - "ForceLoad" + "DebugWaitFunctionNode" }; - com.yahoo.system.ForceLoad.forceLoad(pkg, classes); + com.yahoo.system.ForceLoad.forceLoad(pkg, classes, + ForceLoad.class.getClassLoader()); } public static boolean forceLoad() { diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index 5e248dc8758..1fd69e03048 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -40,7 +40,7 @@ typedef std::unique_ptr<AttributeVector::SearchContext> SearchContextPtr; typedef std::unique_ptr<search::queryeval::SearchIterator> SearchBasePtr; void -toStr(std::stringstream &ss, SearchIterator &it) +toStr(std::stringstream &ss, SearchIterator &it, TermFieldMatchData *md) { it.initFullRange(); it.seek(1u); @@ -51,16 +51,21 @@ toStr(std::stringstream &ss, SearchIterator &it) else ss << ","; ss << it.getDocId(); + if (md != nullptr) { + it.unpack(it.getDocId()); + ss << "[w=" << md->begin()->getElementWeight() << "]"; + } it.seek(it.getDocId() + 1); } } bool -assertIterator(const std::string &exp, SearchIterator &it) +assertIterator(const std::string &exp, SearchIterator &it, + TermFieldMatchData *md = nullptr) { std::stringstream ss; - toStr(ss, it); + toStr(ss, it, md); if (!EXPECT_EQUAL(exp, ss.str())) return false; return true; @@ -140,6 +145,13 @@ private: bool assertSearch(const std::string &exp, StringAttribute &sa); + bool + assertSearch(const std::string &exp, StringAttribute &v, + const std::string &key); + + bool + assertSearch(const std::string &exp, IntegerAttribute &v, int32_t key); + void addDocs(const AttributePtr & ptr, uint32_t numDocs); template <typename VectorType, typename BufferType, typename Range> @@ -175,6 +187,10 @@ private: void testStringFold(void); + + void testDupValuesInIntArray(); + + void testDupValuesInStringArray(); public: int Main(); }; @@ -394,6 +410,37 @@ PostingListAttributeTest::assertSearch(const std::string &exp, } +bool +PostingListAttributeTest::assertSearch(const std::string &exp, + StringAttribute &sa, + const std::string &key) +{ + TermFieldMatchData md; + SearchContextPtr sc = getSearch<StringAttribute, std::string> + (sa, key, false); + sc->fetchPostings(true); + SearchBasePtr sb = sc->createIterator(&md, true); + if (!EXPECT_TRUE(assertIterator(exp, *sb, &md))) + return false; + return true; +} + +bool +PostingListAttributeTest::assertSearch(const std::string &exp, + IntegerAttribute &ia, + int32_t key) +{ + TermFieldMatchData md; + SearchContextPtr sc = getSearch<IntegerAttribute, int32_t> + (ia, key, false); + sc->fetchPostings(true); + SearchBasePtr sb = sc->createIterator(&md, true); + if (!EXPECT_TRUE(assertIterator(exp, *sb, &md))) + return false; + return true; +} + + void PostingListAttributeTest::addDocs(const AttributePtr & ptr, uint32_t numDocs) { @@ -1001,6 +1048,66 @@ PostingListAttributeTest::testStringFold(void) EXPECT_TRUE(assertSearch("", sa)); } +void +PostingListAttributeTest::testDupValuesInIntArray() +{ + Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); + cfg.setFastSearch(true); + AttributePtr ptr1 = AttributeFactory::createAttribute("aint32_3", cfg); + addDocs(ptr1, 6); + + IntegerAttribute &ia(asInt(ptr1)); + + ia.append(1, 1, 1); + ia.append(1, 1, 1); + ia.append(2, 1, 1); + ia.commit(); + EXPECT_TRUE(assertSearch("1[w=2],2[w=1]", ia, 1)); + + ia.clearDoc(1); + ia.append(1, 1, 1); + ia.clearDoc(2); + ia.append(2, 1, 1); + ia.append(2, 1, 1); + ia.commit(); + EXPECT_TRUE(assertSearch("1[w=1],2[w=2]", ia, 1)); +} + +void +PostingListAttributeTest::testDupValuesInStringArray() +{ + Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); + cfg.setFastSearch(true); + AttributePtr ptr1 = AttributeFactory::createAttribute("astr_3", cfg); + addDocs(ptr1, 6); + + StringAttribute &sa(asString(ptr1)); + + sa.append(1, "foo", 1); + sa.append(1, "foo", 1); + sa.append(2, "foo", 1); + sa.append(3, "bar", 1); + sa.append(3, "BAR", 1); + sa.append(4, "bar", 1); + sa.commit(); + EXPECT_TRUE(assertSearch("1[w=2],2[w=1]", sa, "foo")); + EXPECT_TRUE(assertSearch("3[w=2],4[w=1]", sa, "bar")); + + sa.clearDoc(1); + sa.append(1, "foo", 1); + sa.clearDoc(2); + sa.append(2, "foo", 1); + sa.append(2, "foo", 1); + sa.clearDoc(3); + sa.append(3, "bar", 1); + sa.clearDoc(4); + sa.append(4, "bar", 1); + sa.append(4, "BAR", 1); + sa.commit(); + EXPECT_TRUE(assertSearch("1[w=1],2[w=2]", sa, "foo")); + EXPECT_TRUE(assertSearch("3[w=1],4[w=2]", sa, "bar")); +} + int PostingListAttributeTest::Main() @@ -1012,6 +1119,8 @@ PostingListAttributeTest::Main() testReload(); testMinMax(); testStringFold(); + testDupValuesInIntArray(); + testDupValuesInStringArray(); TEST_DONE(); } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 137f93bcffe..73319a30888 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -214,4 +214,12 @@ TEST_F("Test tensortype file header tag", Fixture("tensor(x[10])")) EXPECT_EQUAL("tensor(x[10])", header.getTag("tensortype").asString()); } +TEST_F("Require that tensor attribute can provide empty tensor of correct type", Fixture("tensor(x[10])")) +{ + const TensorAttribute &tensorAttr = *f._tensorAttr; + Tensor::UP emptyTensor = tensorAttr.getEmptyTensor(); + EXPECT_EQUAL(emptyTensor->getType(), tensorAttr.getConfig().tensorType()); + EXPECT_EQUAL(emptyTensor->getType(), TensorType::fromSpec("tensor(x[10])")); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index b0bac4b576d..43f3174b994 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -26,6 +26,7 @@ LOG_SETUP("prod_features_test"); #include <vespa/searchlib/features/firstphasefeature.h> #include <vespa/searchlib/features/foreachfeature.h> #include <vespa/searchlib/features/freshnessfeature.h> +#include <vespa/searchlib/features/matchcountfeature.h> #include <vespa/searchlib/features/matchesfeature.h> #include <vespa/searchlib/features/matchfeature.h> #include <vespa/searchlib/features/nowfeature.h> @@ -87,6 +88,7 @@ Test::Main() TEST_DO(testAttribute()); TEST_FLUSH(); TEST_DO(testAttributeMatch()); TEST_FLUSH(); TEST_DO(testCloseness()); TEST_FLUSH(); + TEST_DO(testMatchCount()); TEST_FLUSH(); TEST_DO(testDistance()); TEST_FLUSH(); TEST_DO(testDistanceToPath()); TEST_FLUSH(); TEST_DO(testDotProduct()); TEST_FLUSH(); @@ -1457,6 +1459,54 @@ Test::testMatch() } void +Test::testMatchCount() +{ + { // Test blueprint. + MatchCountBlueprint pt; + + EXPECT_TRUE(assertCreateInstance(pt, "matchCount")); + + FtFeatureTest ft(_factory, ""); + ft.getIndexEnv().getBuilder().addField(FieldType::INDEX, CollectionType::SINGLE, "foo"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); + + StringList params, in, out; + FT_SETUP_FAIL(pt, ft.getIndexEnv(), params); // expects 1 parameter + FT_SETUP_FAIL(pt, ft.getIndexEnv(), params.add("baz")); // cannot find the field + FT_SETUP_OK(pt, ft.getIndexEnv(), params.clear().add("foo"), in, out.add("out")); + FT_SETUP_OK(pt, ft.getIndexEnv(), params.clear().add("bar"), in, out); + + FT_DUMP_EMPTY(_factory, "matchCount"); + } + { // Test executor for index fields + EXPECT_TRUE(assertMatches(0, "x", "a", "matchCount(foo)")); + EXPECT_TRUE(assertMatches(1, "a", "a", "matchCount(foo)")); + EXPECT_TRUE(assertMatches(2, "a b", "a b", "matchCount(foo)")); + // change docId to indicate no matches in the field + EXPECT_TRUE(assertMatches(0, "a", "a", "matchCount(foo)", 2)); + } + { // Test executor for attribute fields + FtFeatureTest ft(_factory, StringList().add("matchCount(foo)"). + add("matchCount(baz)")); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz"); + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo + ASSERT_TRUE(ft.setup()); + + MatchDataBuilder::UP mdb = ft.createMatchDataBuilder(); + mdb->setWeight("foo", 0, 0); + mdb->setWeight("bar", 1, 0); + mdb->setWeight("foo", 2, 0); + mdb->apply(1); + EXPECT_TRUE(ft.execute(RankResult().addScore("matchCount(foo)", 2))); + EXPECT_TRUE(ft.execute(RankResult().addScore("matchCount(baz)", 0))); + } +} + +void Test::testMatches() { { // Test blueprint. @@ -1479,9 +1529,9 @@ Test::testMatches() FT_DUMP_EMPTY(_factory, "matches"); } { // Test executor for index fields - EXPECT_TRUE(assertMatches(0, "x", "a")); - EXPECT_TRUE(assertMatches(1, "a", "a")); - EXPECT_TRUE(assertMatches(1, "a b", "a b")); + EXPECT_TRUE(assertMatches(0, "x", "a", "matches(foo)")); + EXPECT_TRUE(assertMatches(1, "a", "a", "matches(foo)")); + EXPECT_TRUE(assertMatches(1, "a b", "a b", "matches(foo)")); // change docId to indicate no matches in the field EXPECT_TRUE(assertMatches(0, "a", "a", "matches(foo)", 2)); // specify termIdx as second parameter diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h index dd15981af1f..00c1fa46ec8 100644 --- a/searchlib/src/tests/features/prod_features.h +++ b/searchlib/src/tests/features/prod_features.h @@ -16,6 +16,7 @@ public: void testAttribute(); void testAttributeMatch(); void testCloseness(); + void testMatchCount(); void testDistance(); void testDistanceToPath(); void testDotProduct(); @@ -37,108 +38,40 @@ public: void testUtils(); private: - void - testFieldMatchBluePrint(); - - void - testFieldMatchExecutor(); - - void - testFieldMatchExecutorOutOfOrder(); - - void - testFieldMatchExecutorSegments(); - - void - testFieldMatchExecutorGaps(); - - void - testFieldMatchExecutorHead(); - - void - testFieldMatchExecutorTail(); - - void - testFieldMatchExecutorLongestSequence(); - - void - testFieldMatchExecutorMatches(); - - void - testFieldMatchExecutorCompleteness(); - - void - testFieldMatchExecutorOrderness(); - - void - testFieldMatchExecutorRelatedness(); - - void - testFieldMatchExecutorLongestSequenceRatio(); - - void - testFieldMatchExecutorEarliness(); - - void - testFieldMatchExecutorWeight(); - - void - testFieldMatchExecutorSignificance(); - - void - testFieldMatchExecutorImportance(); - - void - testFieldMatchExecutorOccurrence(); - - void - testFieldMatchExecutorAbsoluteOccurrence(); - - void - testFieldMatchExecutorWeightedOccurrence(); - - void - testFieldMatchExecutorWeightedAbsoluteOccurrence(); - - void - testFieldMatchExecutorSignificantOccurrence(); - - void - testFieldMatchExecutorUnweightedProximity(); - - void - testFieldMatchExecutorReverseProximity(); - - void - testFieldMatchExecutorAbsoluteProximity(); - - void - testFieldMatchExecutorMultiSegmentProximity(); - - void - testFieldMatchExecutorSegmentDistance(); - - void - testFieldMatchExecutorSegmentProximity(); - - void - testFieldMatchExecutorSegmentStarts(); - - void - testFieldMatchExecutorMoreThanASegmentLengthOfUnmatchedQuery(); - - void - testFieldMatchExecutorQueryRepeats(); - - void - testFieldMatchExecutorZeroCases(); - - void - testFieldMatchExecutorExceedingIterationLimit(); - - void - testFieldMatchExecutorRemaining(); - + void testFieldMatchBluePrint(); + void testFieldMatchExecutor(); + void testFieldMatchExecutorOutOfOrder(); + void testFieldMatchExecutorSegments(); + void testFieldMatchExecutorGaps(); + void testFieldMatchExecutorHead(); + void testFieldMatchExecutorTail(); + void testFieldMatchExecutorLongestSequence(); + void testFieldMatchExecutorMatches(); + void testFieldMatchExecutorCompleteness(); + void testFieldMatchExecutorOrderness(); + void testFieldMatchExecutorRelatedness(); + void testFieldMatchExecutorLongestSequenceRatio(); + void testFieldMatchExecutorEarliness(); + void testFieldMatchExecutorWeight(); + void testFieldMatchExecutorSignificance(); + void testFieldMatchExecutorImportance(); + void testFieldMatchExecutorOccurrence(); + void testFieldMatchExecutorAbsoluteOccurrence(); + void testFieldMatchExecutorWeightedOccurrence(); + void testFieldMatchExecutorWeightedAbsoluteOccurrence(); + void testFieldMatchExecutorSignificantOccurrence(); + void testFieldMatchExecutorUnweightedProximity(); + void testFieldMatchExecutorReverseProximity(); + void testFieldMatchExecutorAbsoluteProximity(); + void testFieldMatchExecutorMultiSegmentProximity(); + void testFieldMatchExecutorSegmentDistance(); + void testFieldMatchExecutorSegmentProximity(); + void testFieldMatchExecutorSegmentStarts(); + void testFieldMatchExecutorMoreThanASegmentLengthOfUnmatchedQuery(); + void testFieldMatchExecutorQueryRepeats(); + void testFieldMatchExecutorZeroCases(); + void testFieldMatchExecutorExceedingIterationLimit(); + void testFieldMatchExecutorRemaining(); void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime); void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime); diff --git a/searchlib/src/tests/features/prod_features_test.sh b/searchlib/src/tests/features/prod_features_test.sh index bec2b49807f..765e91400ae 100755 --- a/searchlib/src/tests/features/prod_features_test.sh +++ b/searchlib/src/tests/features/prod_features_test.sh @@ -1,3 +1,4 @@ -#!/bin/bash +#!/bin/sh +set -e VESPA_LOG_TARGET=file:vlog2.txt $VALGRIND ./searchlib_prod_features_test_app rm -rf *.dat diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index caceea0f47b..8bcbb5954ff 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -6,18 +6,19 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/tensorattribute.h> +#include <vespa/searchlib/attribute/tensorattribute.h> #include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/searchlib/fef/test/ftlib.h> -#include <vespa/searchlib/fef/fef.h> -#include <vespa/vespalib/tensor/tensor_factory.h> -#include <vespa/vespalib/tensor/default_tensor.h> -#include <vespa/vespalib/tensor/serialization/typed_binary_format.h> -#include <vespa/searchlib/attribute/tensorattribute.h> #include <vespa/vespalib/eval/interpreted_function.h> +#include <vespa/vespalib/tensor/default_tensor.h> #include <vespa/vespalib/tensor/default_tensor_engine.h> +#include <vespa/vespalib/tensor/serialization/typed_binary_format.h> +#include <vespa/vespalib/tensor/tensor_factory.h> +#include <vespa/vespalib/tensor/tensor_mapper.h> using search::feature_t; using namespace search::fef; @@ -27,16 +28,17 @@ using namespace search::features; using search::AttributeFactory; using search::attribute::TensorAttribute; using search::AttributeVector; -using vespalib::eval::Value; using vespalib::eval::Function; +using vespalib::eval::InterpretedFunction; +using vespalib::eval::Value; +using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::DenseTensorCells; using vespalib::tensor::Tensor; using vespalib::tensor::TensorCells; -using vespalib::tensor::DenseTensorCells; using vespalib::tensor::TensorDimensions; using vespalib::tensor::TensorFactory; +using vespalib::tensor::TensorMapper; using vespalib::tensor::TensorType; -using vespalib::eval::InterpretedFunction; -using vespalib::tensor::DefaultTensorEngine; typedef search::attribute::Config AVC; typedef search::attribute::BasicType AVBT; @@ -162,12 +164,30 @@ struct AsTensor { InterpretedFunction ifun; InterpretedFunction::Context ctx; const Value *result; + const Tensor *tensor; explicit AsTensor(const vespalib::string &expr) : ifun(DefaultTensorEngine::ref(), Function::parse(expr)), ctx(), result(&ifun.eval(ctx)) { ASSERT_TRUE(result->is_tensor()); + tensor = static_cast<const Tensor *>(result->as_tensor()); + } + bool operator==(const Tensor &rhs) const { + return tensor->equals(rhs); + } +}; + +struct AsEmptyTensor : public AsTensor { + TensorMapper mapper; + Tensor::UP mappedTensor; + AsEmptyTensor(const vespalib::string &type) + : AsTensor("{ }"), + mapper(TensorType::fromSpec(type)), + mappedTensor(mapper.map(*tensor)) + {} + bool operator==(const Tensor &rhs) const { + return mappedTensor->equals(rhs); } - bool operator==(const Tensor &rhs) const { return static_cast<const Tensor &>(*result->as_tensor()).equals(rhs); } + }; std::ostream &operator<<(std::ostream &os, const AsTensor &my_tensor) { @@ -190,24 +210,24 @@ TEST_F("require that tensor from query can be extracted as tensor in query featu TEST_F("require that empty tensor is created if attribute does not exists", ExecFixture("attribute(null)")) { - EXPECT_EQUAL(AsTensor("{ }"), f.execute()); + EXPECT_EQUAL(AsEmptyTensor("tensor(x{})"), f.execute()); } TEST_F("require that empty tensor is created if tensor type is wrong", ExecFixture("attribute(wrongtype)")) { - EXPECT_EQUAL(AsTensor("{ }"), f.execute()); + EXPECT_EQUAL(AsEmptyTensor("tensor(x{})"), f.execute()); } TEST_F("require that empty tensor is created if query parameter is not found", ExecFixture("query(null)")) { - EXPECT_EQUAL(AsTensor("{ }"), f.execute()); + EXPECT_EQUAL(AsEmptyTensor("tensor(q{})"), f.execute()); } -TEST_F("require that empty tensor is created if document has no tensor", +TEST_F("require that empty tensor with correct type is created if document has no tensor", ExecFixture("attribute(tensorattr)")) { - EXPECT_EQUAL(AsTensor("{ }"), f.execute(2)); + EXPECT_EQUAL(AsEmptyTensor("tensor(x{})"), f.execute(2)); } struct AsDenseTensor { diff --git a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp b/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp index 75ad526e2f7..351d19737aa 100644 --- a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp +++ b/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp @@ -1,44 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP("fileheaderinspect_test"); +#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/util/fileheadertk.h> -#include <vespa/vespalib/testkit/testapp.h> using namespace search; using namespace vespalib; -class Test : public vespalib::TestApp { -private: - bool writeHeader(const FileHeader &header, const vespalib::string &fileName); - vespalib::string readFile(const vespalib::string &fileName); - - void testError(); - void testEscape(); - void testDelimiter(); - void testQuiet(); - void testVerbose(); - -public: - int Main() { - TEST_INIT("fileheaderinspect_test"); - - testError(); TEST_FLUSH(); - testEscape(); TEST_FLUSH(); - testDelimiter(); TEST_FLUSH(); - testQuiet(); TEST_FLUSH(); - testVerbose(); TEST_FLUSH(); - - TEST_DONE(); - } -}; - -TEST_APPHOOK(Test); - -bool -Test::writeHeader(const FileHeader &header, const vespalib::string &fileName) -{ +bool writeHeader(const FileHeader &header, const vespalib::string &fileName) { FastOS_File file; if (!EXPECT_TRUE(file.OpenWriteOnlyTruncate(fileName.c_str()))) { return false; @@ -50,30 +18,24 @@ Test::writeHeader(const FileHeader &header, const vespalib::string &fileName) return true; } -vespalib::string -Test::readFile(const vespalib::string &fileName) -{ +vespalib::string readFile(const vespalib::string &fileName) { FastOS_File file; ASSERT_TRUE(file.OpenReadOnly(fileName.c_str())); - char buf[1024]; - uint32_t len = file.Read(buf, 1024); - EXPECT_TRUE(len != 1024); // make sure we got everything + char buf[4096]; + uint32_t len = file.Read(buf, sizeof(buf)); + EXPECT_LESS(len, sizeof(buf)); // make sure we got everything vespalib::string str(buf, len); file.Close(); return str; } -void -Test::testError() -{ +TEST("testError") { EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect notfound.dat") != 0); } -void -Test::testEscape() -{ +TEST("testEscape") { FileHeader header; header.putTag(FileHeader::Tag("fanart", "\fa\na\r\t")); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); @@ -81,9 +43,7 @@ Test::testEscape() EXPECT_EQUAL("fanart;string;\\fa\\na\\r\\t\n", readFile("out")); } -void -Test::testDelimiter() -{ +TEST("testDelimiter") { FileHeader header; header.putTag(FileHeader::Tag("string", "string")); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); @@ -91,41 +51,39 @@ Test::testDelimiter() EXPECT_EQUAL("str\\ingistr\\ingistr\\ing\n", readFile("out")); } -void -Test::testVerbose() -{ +TEST("testQuiet") { FileHeader header; FileHeaderTk::addVersionTags(header); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0); vespalib::string str = readFile("out"); EXPECT_TRUE(!str.empty()); for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) { const FileHeader::Tag &tag = header.getTag(i); - EXPECT_TRUE(str.find(tag.getName()) != vespalib::string::npos); + size_t pos = str.find(tag.getName()); + EXPECT_TRUE(pos != vespalib::string::npos); vespalib::asciistream out; - out << tag; - EXPECT_TRUE(str.find(out.str()) != vespalib::string::npos); + out << ";" << tag; + EXPECT_TRUE(str.find(out.str(), pos) != vespalib::string::npos); } } -void -Test::testQuiet() -{ +TEST("testVerbose") { FileHeader header; FileHeaderTk::addVersionTags(header); ASSERT_TRUE(writeHeader(header, "fileheader.dat")); - EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0); + EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect fileheader.dat > out") == 0); vespalib::string str = readFile("out"); EXPECT_TRUE(!str.empty()); for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) { const FileHeader::Tag &tag = header.getTag(i); - size_t pos = str.find(tag.getName()); - EXPECT_TRUE(pos != vespalib::string::npos); + EXPECT_TRUE(str.find(tag.getName()) != vespalib::string::npos); vespalib::asciistream out; - out << ";" << tag; - EXPECT_TRUE(str.find(out.str(), pos) != vespalib::string::npos); + out << tag; + EXPECT_TRUE(str.find(out.str()) != vespalib::string::npos); } } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/blueprint/.gitignore b/searchlib/src/tests/queryeval/blueprint/.gitignore index da4bf633103..9bd9a86271b 100644 --- a/searchlib/src/tests/queryeval/blueprint/.gitignore +++ b/searchlib/src/tests/queryeval/blueprint/.gitignore @@ -6,3 +6,4 @@ rhs.out searchlib_blueprint_test_app searchlib_intermediate_blueprints_test_app searchlib_leaf_blueprints_test_app +/index diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 775654d23fc..2e756e9d4b8 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -715,7 +715,7 @@ void Test::testErase() "count %zu, numBytes %zu", partId, (uint64_t) part.range.from(), (uint64_t) part.range.to(), - part.count, part.byteSize); + part.numEntries, part.byteSize); } ASSERT_LESS_EQUAL(2u, numParts); // Erase everything before second to last domainpart file @@ -726,8 +726,8 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 2].count + - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 2].numEntries + + domainInfo.parts[numParts - 1].numEntries)); // No apparent effect of erasing just first entry in 2nd to last part s1->erase(eraseSerial + 1); TEST_DO(assertVisitStats(tls, "erase", 2, TOTAL_NUM_ENTRIES, @@ -735,8 +735,8 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial + 1, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 2].count + - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 2].numEntries + + domainInfo.parts[numParts - 1].numEntries)); // No apparent effect of erasing almost all of 2nd to last part SerialNum eraseSerial2 = domainInfo.parts[numParts - 2].range.to(); s1->erase(eraseSerial2); @@ -745,8 +745,8 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial2, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 2].count + - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 2].numEntries + + domainInfo.parts[numParts - 1].numEntries)); // Erase everything before last domainpart file eraseSerial = domainInfo.parts[numParts - 1].range.from(); s1->erase(eraseSerial); @@ -755,7 +755,7 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 1].numEntries)); // No apparent effect of erasing just first entry in last part s1->erase(eraseSerial + 1); TEST_DO(assertVisitStats(tls, "erase", 2, TOTAL_NUM_ENTRIES, @@ -763,7 +763,7 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial + 1, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 1].numEntries)); // No apparent effect of erasing almost all of last part eraseSerial2 = domainInfo.parts[numParts - 1].range.to(); s1->erase(eraseSerial2); @@ -772,7 +772,7 @@ void Test::testErase() TOTAL_NUM_ENTRIES + 1 - eraseSerial, TOTAL_NUM_ENTRIES - eraseSerial)); TEST_DO(assertStatus(*s1, eraseSerial2, TOTAL_NUM_ENTRIES, - domainInfo.parts[numParts - 1].count)); + domainInfo.parts[numParts - 1].numEntries)); } } diff --git a/searchlib/src/tests/transactionlogstress/.gitignore b/searchlib/src/tests/transactionlogstress/.gitignore index 5913613b455..bdfe824be8b 100644 --- a/searchlib/src/tests/transactionlogstress/.gitignore +++ b/searchlib/src/tests/transactionlogstress/.gitignore @@ -2,3 +2,4 @@ Makefile server translogstress +/searchlib_translogstress_app diff --git a/searchlib/src/tests/url/testurl.cpp b/searchlib/src/tests/url/testurl.cpp index 4ed28453890..bfe6c79de07 100644 --- a/searchlib/src/tests/url/testurl.cpp +++ b/searchlib/src/tests/url/testurl.cpp @@ -580,7 +580,7 @@ int main(int, char **) "SCHEME:http,DOMAIN:sony,DOMAIN:co,MAINTLD:uk");// Tokenstring #endif // Test fixes for bugs reported in cvs commit: - // tegge 2000/10/27 22:42:59 CEST + // toregge 2000/10/27 22:42:59 CEST success = success && CheckURL("http://somehost.somedomain/this!is!it/boom", // URL "http", // scheme diff --git a/searchlib/src/vespa/searchlib/CMakeLists.txt b/searchlib/src/vespa/searchlib/CMakeLists.txt index f0351687918..0f0420e5b61 100644 --- a/searchlib/src/vespa/searchlib/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/CMakeLists.txt @@ -2,31 +2,35 @@ vespa_add_library(searchlib SOURCES $<TARGET_OBJECTS:searchlib_aggregation> - $<TARGET_OBJECTS:searchlib_grouping> $<TARGET_OBJECTS:searchlib_attribute> + $<TARGET_OBJECTS:searchlib_bitcompression> $<TARGET_OBJECTS:searchlib_btree> $<TARGET_OBJECTS:searchlib_common> + $<TARGET_OBJECTS:searchlib_diskindex> $<TARGET_OBJECTS:searchlib_docstore> $<TARGET_OBJECTS:searchlib_engine> $<TARGET_OBJECTS:searchlib_expression> + $<TARGET_OBJECTS:searchlib_features> + $<TARGET_OBJECTS:searchlib_features_fieldmatch> + $<TARGET_OBJECTS:searchlib_features_rankingexpression> $<TARGET_OBJECTS:searchlib_fef> $<TARGET_OBJECTS:searchlib_fef_test> $<TARGET_OBJECTS:searchlib_fef_test_plugin> + $<TARGET_OBJECTS:searchlib_grouping> + $<TARGET_OBJECTS:searchlib_memoryindex> $<TARGET_OBJECTS:searchlib_parsequery> $<TARGET_OBJECTS:searchlib_predicate> + $<TARGET_OBJECTS:searchlib_query> + $<TARGET_OBJECTS:searchlib_query_tree> + $<TARGET_OBJECTS:searchlib_queryeval> + $<TARGET_OBJECTS:searchlib_queryeval_wand> $<TARGET_OBJECTS:searchlib_sconfig> - $<TARGET_OBJECTS:searchlib_searchlib_bitcompression> - $<TARGET_OBJECTS:searchlib_searchlib_diskindex> $<TARGET_OBJECTS:searchlib_searchlib_index> - $<TARGET_OBJECTS:searchlib_searchlib_memoryindex> - $<TARGET_OBJECTS:searchlib_translog> + $<TARGET_OBJECTS:searchlib_transactionlog> $<TARGET_OBJECTS:searchlib_util> + INSTALL lib64 DEPENDS - searchlib_features - searchlib_query - searchlib_queryeval - searchlib_queryeval_test staging_vespalib icuuc atomic diff --git a/searchlib/src/vespa/searchlib/attribute/OWNERS b/searchlib/src/vespa/searchlib/attribute/OWNERS index fda83fc8cf8..23c5db50ab7 100644 --- a/searchlib/src/vespa/searchlib/attribute/OWNERS +++ b/searchlib/src/vespa/searchlib/attribute/OWNERS @@ -1,3 +1,3 @@ -tegge +toregge geirst baldersheim diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 4db2ce1da2c..62be248d60a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -32,6 +32,7 @@ class StringEnumIndexMapper : public EnumIndexMapper public: StringEnumIndexMapper(const EnumPostingTree & dictionary) : _dictionary(dictionary) { } EnumStoreBase::Index map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const override; + virtual bool hasFold() const override { return true; } private: const EnumPostingTree & _dictionary; }; diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index 2731fb0157d..15da16e73fb 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -159,44 +159,148 @@ PostingChange<P>::apply(GrowableBitVector &bv) template <typename WeightedIndex> class ActualChangeComputer { public: - typedef std::vector<WeightedIndex> V; + using EnumIndex = EnumStoreBase::Index; + using AlwaysWeightedIndexVector = std::vector<multivalue::WeightedValue<EnumIndex>>; + using WeightedIndexVector = std::vector<WeightedIndex>; void compute(const WeightedIndex * entriesNew, size_t szNew, const WeightedIndex * entriesOld, size_t szOld, - V & added, V & changed, V & removed) const; + AlwaysWeightedIndexVector & added, AlwaysWeightedIndexVector & changed, AlwaysWeightedIndexVector & removed); + + ActualChangeComputer(const EnumStoreComparator &compare, + const EnumIndexMapper &mapper) + : _oldEntries(), + _newEntries(), + _cachedMapping(), + _compare(compare), + _mapper(mapper), + _hasFold(mapper.hasFold()) + { + } + private: - mutable V _oldEntries; - mutable V _newEntries; + WeightedIndexVector _oldEntries; + WeightedIndexVector _newEntries; + vespalib::hash_map<uint32_t, uint32_t> _cachedMapping; + const EnumStoreComparator &_compare; + const EnumIndexMapper &_mapper; + const bool _hasFold; + + static void copyFast(WeightedIndexVector &dst, const WeightedIndex *src, size_t sz) + { + dst.insert(dst.begin(), src, src + sz); + } + + EnumIndex mapEnumIndex(EnumIndex unmapped) { + auto itr = _cachedMapping.insert(std::make_pair(unmapped.ref(), 0)); + if (itr.second) { + itr.first->second = _mapper.map(unmapped, _compare).ref(); + } + return EnumIndex(itr.first->second); + } + + + void copyMapped(WeightedIndexVector &dst, const WeightedIndex *src, size_t sz) + { + const WeightedIndex *srce = src + sz; + for (const WeightedIndex *i = src; i < srce; ++i) { + dst.emplace_back(mapEnumIndex(i->value()), i->weight()); + } + } + + void copyEntries(WeightedIndexVector &dst, const WeightedIndex *src, size_t sz) + { + dst.reserve(sz); + dst.clear(); + if (_hasFold) { + copyMapped(dst, src, sz); + } else { + copyFast(dst, src, sz); + } + std::sort(dst.begin(), dst.end()); + } +}; + +template <typename WeightedIndex> +class MergeDupIterator { + using InnerIter = typename std::vector<WeightedIndex>::const_iterator; + using EnumIndex = EnumStoreBase::Index; + using Entry = multivalue::WeightedValue<EnumIndex>; + InnerIter _cur; + InnerIter _end; + Entry _entry; + bool _valid; + void merge() { + EnumIndex idx = _cur->value(); + int32_t weight = _cur->weight(); + ++_cur; + while (_cur != _end && _cur->value() == idx) { + // sum weights together. Overflow is not handled. + weight += _cur->weight(); + ++_cur; + } + _entry = Entry(idx, weight); + } +public: + MergeDupIterator(const std::vector<WeightedIndex> &vec) + : _cur(vec.begin()), + _end(vec.end()), + _entry(), + _valid(_cur != _end) + { + if (_valid) { + merge(); + } + } + + bool valid() const { return _valid; } + const Entry &entry() const { return _entry; } + EnumIndex value() const { return _entry.value(); } + int32_t weight() const { return _entry.weight(); } + void next() { + if (_cur != _end) { + merge(); + } else { + _valid = false; + } + } }; template <typename WeightedIndex> void ActualChangeComputer<WeightedIndex>::compute(const WeightedIndex * entriesNew, size_t szNew, const WeightedIndex * entriesOld, size_t szOld, - V & added, V & changed, V & removed) const + AlwaysWeightedIndexVector & added, + AlwaysWeightedIndexVector & changed, + AlwaysWeightedIndexVector & removed) { - _newEntries.reserve(szNew); - _oldEntries.reserve(szOld); - _newEntries.clear(); - _oldEntries.clear(); - _newEntries.insert(_newEntries.begin(), entriesNew, entriesNew + szNew); - _oldEntries.insert(_oldEntries.begin(), entriesOld, entriesOld + szOld); - std::sort(_newEntries.begin(), _newEntries.end()); - std::sort(_oldEntries.begin(), _oldEntries.end()); - auto newIt(_newEntries.begin()), oldIt(_oldEntries.begin()); - while (newIt != _newEntries.end() && oldIt != _oldEntries.end()) { - if (newIt->value() == oldIt->value()) { - if (newIt->weight() != oldIt->weight()) { - changed.push_back(*newIt); + copyEntries(_newEntries, entriesNew, szNew); + copyEntries(_oldEntries, entriesOld, szOld); + MergeDupIterator<WeightedIndex> oldIt(_oldEntries); + MergeDupIterator<WeightedIndex> newIt(_newEntries); + + while (newIt.valid() && oldIt.valid()) { + if (newIt.value() == oldIt.value()) { + if (newIt.weight() != oldIt.weight()) { + changed.push_back(newIt.entry()); } - newIt++, oldIt++; - } else if (newIt->value() < oldIt->value()) { - added.push_back(*newIt++); + newIt.next(); + oldIt.next(); + } else if (newIt.value() < oldIt.value()) { + added.push_back(newIt.entry()); + newIt.next(); } else { - removed.push_back(*oldIt++); + removed.push_back(oldIt.entry()); + oldIt.next(); } } - added.insert(added.end(), newIt, _newEntries.end()); - removed.insert(removed.end(), oldIt, _oldEntries.end()); + while (newIt.valid()) { + added.push_back(newIt.entry()); + newIt.next(); + } + while (oldIt.valid()) { + removed.push_back(oldIt.entry()); + oldIt.next(); + } } template <typename WeightedIndex, typename PostingMap> @@ -207,8 +311,8 @@ compute(const MultivalueMapping & mvm, const DocIndices & docIndices, const EnumStoreComparator & compare, const EnumIndexMapper & mapper) { typedef ActualChangeComputer<WeightedIndex> AC; - AC actualChange; - typename AC::V added, changed, removed; + AC actualChange(compare, mapper); + typename AC::AlwaysWeightedIndexVector added, changed, removed; PostingMap changePost; // generate add postings and remove postings @@ -218,14 +322,14 @@ compute(const MultivalueMapping & mvm, const DocIndices & docIndices, added.clear(), changed.clear(), removed.clear(); actualChange.compute(&docIndex.second[0], docIndex.second.size(), oldIndices, valueCount, added, changed, removed); - for (const WeightedIndex & wi : added) { - changePost[EnumPostingPair(mapper.map(wi.value(), compare), &compare)].add(docIndex.first, wi.weight()); + for (const auto & wi : added) { + changePost[EnumPostingPair(wi.value(), &compare)].add(docIndex.first, wi.weight()); } - for (const WeightedIndex & wi : removed) { - changePost[EnumPostingPair(mapper.map(wi.value(), compare), &compare)].remove(docIndex.first); + for (const auto & wi : removed) { + changePost[EnumPostingPair(wi.value(), &compare)].remove(docIndex.first); } - for (const WeightedIndex & wi : changed) { - changePost[EnumPostingPair(mapper.map(wi.value(), compare), &compare)].remove(docIndex.first).add(docIndex.first, wi.weight()); + for (const auto & wi : changed) { + changePost[EnumPostingPair(wi.value(), &compare)].remove(docIndex.first).add(docIndex.first, wi.weight()); } } return changePost; diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.h b/searchlib/src/vespa/searchlib/attribute/postingchange.h index 8309cf91516..b4abe0c7dd2 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.h +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.h @@ -48,6 +48,7 @@ class EnumIndexMapper public: virtual ~EnumIndexMapper() { } virtual EnumStoreBase::Index map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const; + virtual bool hasFold() const { return false; } }; template <typename WeightedIndex, typename PostingMap> diff --git a/searchlib/src/vespa/searchlib/attribute/tensorattribute.cpp b/searchlib/src/vespa/searchlib/attribute/tensorattribute.cpp index 0e83749847f..6e504d8cfec 100644 --- a/searchlib/src/vespa/searchlib/attribute/tensorattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/tensorattribute.cpp @@ -2,10 +2,12 @@ #include <vespa/fastos/fastos.h> #include "tensorattribute.h" +#include <vespa/vespalib/tensor/default_tensor.h> #include <vespa/vespalib/tensor/tensor.h> #include "tensorattributesaver.h" using vespalib::tensor::Tensor; +using vespalib::tensor::TensorMapper; namespace search { @@ -33,6 +35,13 @@ public: void readTensor(void *buf, size_t len) { _datFile->ReadBuf(buf, len); } }; +Tensor::UP +createEmptyTensor(const TensorMapper &mapper) +{ + vespalib::tensor::DefaultTensor::builder builder; + return mapper.map(*builder.build()); +} + } TensorAttribute::TensorAttribute(const vespalib::stringref &baseFileName, @@ -185,6 +194,11 @@ TensorAttribute::getTensor(DocId docId) const return _tensorStore.getTensor(ref); } +Tensor::UP +TensorAttribute::getEmptyTensor() const +{ + return createEmptyTensor(_tensorMapper); +} void TensorAttribute::clearDocs(DocId lidLow, DocId lidLimit) diff --git a/searchlib/src/vespa/searchlib/attribute/tensorattribute.h b/searchlib/src/vespa/searchlib/attribute/tensorattribute.h index 954d211d13f..27294cb5527 100644 --- a/searchlib/src/vespa/searchlib/attribute/tensorattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/tensorattribute.h @@ -41,6 +41,7 @@ public: virtual bool addDoc(DocId &docId) override; void setTensor(DocId docId, const Tensor &tensor); std::unique_ptr<Tensor> getTensor(DocId docId) const; + std::unique_ptr<Tensor> getEmptyTensor() const; virtual void clearDocs(DocId lidLow, DocId lidLimit) override; virtual void onShrinkLidSpace() override; virtual bool onLoad() override; diff --git a/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt b/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt index 51d299bacfa..d639bed05bf 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_bitcompression OBJECT +vespa_add_library(searchlib_bitcompression OBJECT SOURCES compression.cpp countcompression.cpp diff --git a/searchlib/src/vespa/searchlib/bitcompression/OWNERS b/searchlib/src/vespa/searchlib/bitcompression/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/OWNERS +++ b/searchlib/src/vespa/searchlib/bitcompression/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchlib/src/vespa/searchlib/btree/OWNERS b/searchlib/src/vespa/searchlib/btree/OWNERS index e6340232840..b7b549c6058 100644 --- a/searchlib/src/vespa/searchlib/btree/OWNERS +++ b/searchlib/src/vespa/searchlib/btree/OWNERS @@ -1,2 +1,2 @@ -tegge +toregge geirst diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 1cde63458ec..d9b7237c065 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_diskindex OBJECT +vespa_add_library(searchlib_diskindex OBJECT SOURCES bitvectordictionary.cpp bitvectorfile.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/OWNERS b/searchlib/src/vespa/searchlib/diskindex/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchlib/src/vespa/searchlib/diskindex/OWNERS +++ b/searchlib/src/vespa/searchlib/diskindex/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchlib/src/vespa/searchlib/docstore/OWNERS b/searchlib/src/vespa/searchlib/docstore/OWNERS index d14485102db..f4d47806ed9 100644 --- a/searchlib/src/vespa/searchlib/docstore/OWNERS +++ b/searchlib/src/vespa/searchlib/docstore/OWNERS @@ -1,2 +1,2 @@ baldersheim -tegge +toregge diff --git a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp index 9a1d3639a8e..2694385a557 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/expression/enumresultnode.h> #include <vespa/searchlib/expression/nullresultnode.h> #include <vespa/searchlib/expression/positiveinfinityresultnode.h> +#include <math.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.documentexpressions"); diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index ec21aa87fae..2ce9fc6886c 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_features +vespa_add_library(searchlib_features OBJECT SOURCES agefeature.cpp array_parser.cpp @@ -24,6 +24,7 @@ vespa_add_library(searchlib_features freshnessfeature.cpp item_raw_score_feature.cpp jarowinklerdistancefeature.cpp + matchcountfeature.cpp matchesfeature.cpp matchfeature.cpp native_dot_product_feature.cpp @@ -57,8 +58,5 @@ vespa_add_library(searchlib_features utils.cpp valuefeature.cpp weighted_set_parser.cpp - $<TARGET_OBJECTS:searchlib_fieldmatch> - $<TARGET_OBJECTS:searchlib_rankingexpression> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index b6eb2421ff5..5f3310c4b53 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -391,19 +391,19 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: if (attribute == NULL) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(); + return ConstantTensorExecutor::createEmpty(tensorType); } if (attribute->getCollectionType() != search::attribute::CollectionType::SINGLE || attribute->getBasicType() != search::attribute::BasicType::TENSOR) { LOG(warning, "The attribute vector '%s' is NOT of type tensor." " Returning empty tensor.", attribute->getName().c_str()); - return ConstantTensorExecutor::createEmpty(); + return ConstantTensorExecutor::createEmpty(tensorType); } const TensorAttribute *tensorAttribute = dynamic_cast<const TensorAttribute *>(attribute); if (tensorAttribute == nullptr) { LOG(warning, "The attribute vector '%s' could not be converted to a tensor attribute." " Returning empty tensor.", attribute->getName().c_str()); - return ConstantTensorExecutor::createEmpty(); + return ConstantTensorExecutor::createEmpty(tensorType); } if (tensorType != tensorAttribute->getConfig().tensorType()) { LOG(warning, "The tensor attribute '%s' has tensor type '%s'," @@ -411,7 +411,7 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: tensorAttribute->getName().c_str(), tensorAttribute->getConfig().tensorType().toSpec().c_str(), tensorType.toSpec().c_str()); - return ConstantTensorExecutor::createEmpty(); + return ConstantTensorExecutor::createEmpty(tensorType); } return FeatureExecutor::LP(new TensorFromTensorAttributeExecutor(tensorAttribute)); } diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 11b875df96b..03d423c9510 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -4,8 +4,9 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/vespalib/eval/value.h> -#include <vespa/vespalib/tensor/tensor.h> #include <vespa/vespalib/tensor/default_tensor.h> +#include <vespa/vespalib/tensor/tensor.h> +#include <vespa/vespalib/tensor/tensor_mapper.h> #include <memory> namespace search { @@ -38,6 +39,13 @@ public: (std::make_unique<vespalib::eval::TensorValue> (builder.build()))); } + static fef::FeatureExecutor::LP createEmpty(const vespalib::tensor::TensorType &tensorType) { + vespalib::tensor::DefaultTensor::builder builder; + vespalib::tensor::TensorMapper mapper(tensorType); + return FeatureExecutor::LP(new ConstantTensorExecutor + (std::make_unique<vespalib::eval::TensorValue> + (mapper.map(*builder.build())))); + } }; } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt index 2bbdf179763..965bbcf680e 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_fieldmatch OBJECT +vespa_add_library(searchlib_features_fieldmatch OBJECT SOURCES computer.cpp metrics.cpp diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index 1fc8203a58e..a4dfc4821df 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -68,10 +68,13 @@ ItemRawScoreBlueprint::resolve(const search::fef::IQueryEnvironment &env, { HandleVector handles; const ITermData *term = util::getTermByLabel(env, label); - for (uint32_t i = 0; (term != 0) && (i < term->numFields()); ++i) { - TermFieldHandle handle = term->field(i).getHandle(); - if (handle != IllegalHandle) { - handles.push_back(handle); + if (term != nullptr) { + uint32_t numFields(term->numFields()); + for (uint32_t i(0); i < numFields; ++i) { + TermFieldHandle handle = term->field(i).getHandle(); + if (handle != IllegalHandle) { + handles.push_back(handle); + } } } return handles; diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp new file mode 100644 index 00000000000..b9b84bb97c2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/searchlib/fef/fieldinfo.h> +#include "matchcountfeature.h" +#include "utils.h" +#include "valuefeature.h" + +#include <vespa/log/log.h> +LOG_SETUP(".features.matchcountfeature"); + +using namespace search::fef; + +namespace search { +namespace features { + +MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env) + : FeatureExecutor(), + _handles() +{ + for (uint32_t i = 0; i < env.getNumTerms(); ++i) { + TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId); + if (handle != IllegalHandle) { + _handles.push_back(handle); + } + } +} + +void +MatchCountExecutor::execute(MatchData &match) +{ + size_t output = 0; + for (uint32_t i = 0; i < _handles.size(); ++i) { + const TermFieldMatchData *tfmd = match.resolveTermField(_handles[i]); + if (tfmd->getDocId() == match.getDocId()) { + output++; + } + } + *match.resolveFeature(outputs()[0]) = static_cast<feature_t>(output); +} + + +MatchCountBlueprint::MatchCountBlueprint() : + Blueprint("matchCount"), + _field(NULL) +{ +} + +void +MatchCountBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const +{ +} + +bool +MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & params) +{ + _field = params[0].asField(); + describeOutput("out", "Returns number of matches in the field of all terms in the query"); + return true; +} + +Blueprint::UP +MatchCountBlueprint::createInstance() const +{ + return Blueprint::UP(new MatchCountBlueprint()); +} + +FeatureExecutor::LP +MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv) const +{ + if (_field == nullptr) { + return FeatureExecutor::LP(new ValueExecutor(std::vector<feature_t>(1, 0.0))); + } + return FeatureExecutor::LP(new MatchCountExecutor(_field->id(), queryEnv)); +} + +} // namespace features +} // namespace search diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.h b/searchlib/src/vespa/searchlib/features/matchcountfeature.h new file mode 100644 index 00000000000..2f831dcf9a3 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.h @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/fef/featureexecutor.h> + +namespace search { +namespace features { + +/** + * Implements the executor for the matchCount feature for index and + * attribute fields. + */ +class MatchCountExecutor : public fef::FeatureExecutor +{ +private: + std::vector<fef::TermFieldHandle> _handles; + +public: + MatchCountExecutor(uint32_t fieldId, const fef::IQueryEnvironment &env); + void execute(fef::MatchData & data) override; +}; + +/** + * Implements the blueprint for the matchCount executor. + * + * matchCount(name) + * - returns number of matches of the query in the particular field. + */ +class MatchCountBlueprint : public fef::Blueprint +{ +private: + const fef::FieldInfo *_field; + +public: + MatchCountBlueprint(); + + void visitDumpFeatures(const fef::IIndexEnvironment & env, + fef::IDumpFeatureVisitor & visitor) const override; + + fef::Blueprint::UP createInstance() const override; + + fef::ParameterDescriptions getDescriptions() const override { + return fef::ParameterDescriptions().desc().field(); + } + + bool setup(const fef::IIndexEnvironment & env, + const fef::ParameterList & params) override; + + fef::FeatureExecutor::LP createExecutor(const fef::IQueryEnvironment & env) const override; +}; + +} // namespace features +} // namespace search + diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index e2dbc2d668c..32763ef6ef2 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -132,7 +132,7 @@ createTensorExecutor(const search::fef::IQueryEnvironment &env, } return ConstantTensorExecutor::create(std::move(tensor)); } - return ConstantTensorExecutor::createEmpty(); + return ConstantTensorExecutor::createEmpty(tensorType); } } diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt index 2853a06c49e..7c4ba6a3b9c 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_rankingexpression OBJECT +vespa_add_library(searchlib_features_rankingexpression OBJECT SOURCES feature_name_extractor.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index e05569c0b6d..7c98fc8c1ea 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -24,6 +24,7 @@ #include "freshnessfeature.h" #include "item_raw_score_feature.h" #include "jarowinklerdistancefeature.h" +#include "matchcountfeature.h" #include "matchesfeature.h" #include "matchfeature.h" #include "native_dot_product_feature.h" @@ -63,6 +64,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(Blueprint::SP(new AttributeBlueprint())); registry.addPrototype(Blueprint::SP(new AttributeMatchBlueprint())); registry.addPrototype(Blueprint::SP(new ClosenessBlueprint())); + registry.addPrototype(Blueprint::SP(new MatchCountBlueprint())); registry.addPrototype(Blueprint::SP(new DistanceBlueprint())); registry.addPrototype(Blueprint::SP(new DistanceToPathBlueprint())); registry.addPrototype(Blueprint::SP(new DebugAttributeWaitBlueprint())); diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.cpp index 2e00b5d4f19..36cf0cc3068 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.cpp @@ -12,9 +12,7 @@ TensorFromTensorAttributeExecutor(const search::attribute::TensorAttribute * attribute) : _attribute(attribute), _tensor(), - _builder(), - // XXX: we should use numbers instead of empty tensors - _emptyTensor(std::make_unique<vespalib::eval::TensorValue>(_builder.build())) + _emptyTensor(std::make_unique<vespalib::eval::TensorValue>(attribute->getEmptyTensor())) { } diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.h b/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.h index aa037b4ab59..b486e1ee6bc 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/tensor_from_tensor_attribute_executor.h @@ -17,7 +17,6 @@ class TensorFromTensorAttributeExecutor : public fef::FeatureExecutor private: const search::attribute::TensorAttribute *_attribute; vespalib::eval::TensorValue::UP _tensor; - vespalib::tensor::DefaultTensor::builder _builder; vespalib::eval::TensorValue::UP _emptyTensor; public: diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.h b/searchlib/src/vespa/searchlib/fef/simpletermdata.h index ee4cab468e1..68d6dcd1160 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.h +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.h @@ -14,7 +14,7 @@ namespace fef { /** * Static match data for a single unit (term/phrase/etc). **/ -class SimpleTermData : public ITermData +class SimpleTermData final : public ITermData { private: query::Weight _weight; @@ -40,37 +40,37 @@ public: /** * Returns the term weight. **/ - virtual query::Weight getWeight() const { return _weight; } + query::Weight getWeight() const override { return _weight; } /** * Returns the number of terms represented by this term data object. **/ - virtual uint32_t getPhraseLength() const { return _numTerms; } + uint32_t getPhraseLength() const override { return _numTerms; } /** * Obtain the location of this term in the original user query. * * @return term index **/ - virtual uint32_t getTermIndex() const { return _termIndex; } + uint32_t getTermIndex() const override { return _termIndex; } /** * Obtain the unique id of this term. 0 means not set. * * @return unique id or 0 **/ - virtual uint32_t getUniqueId() const { return _uniqueId; } + uint32_t getUniqueId() const override { return _uniqueId; } /** * Get number of fields searched **/ - virtual size_t numFields() const { return _fields.size(); } + size_t numFields() const override { return _fields.size(); } /** * Direct access to data for individual fields * @param i local index, must have: 0 <= i < numFields() */ - virtual const ITermFieldData &field(size_t i) const { + const ITermFieldData &field(size_t i) const override { return _fields[i]; } @@ -81,7 +81,7 @@ public: * * @return term field data, or NULL if not found **/ - virtual const ITermFieldData *lookupField(uint32_t fieldId) const { + const ITermFieldData *lookupField(uint32_t fieldId) const override { for (size_t fieldIdx(0), m(numFields()); fieldIdx < m; ++fieldIdx) { const ITermFieldData &tfd = field(fieldIdx); if (tfd.getFieldId() == fieldId) { diff --git a/searchlib/src/vespa/searchlib/fef/simpletermfielddata.h b/searchlib/src/vespa/searchlib/fef/simpletermfielddata.h index f95ca5b3472..190ae8e80d2 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermfielddata.h +++ b/searchlib/src/vespa/searchlib/fef/simpletermfielddata.h @@ -38,14 +38,14 @@ public: * * @return field id **/ - virtual uint32_t getFieldId() const { return _fieldId; } + uint32_t getFieldId() const override final { return _fieldId; } /** * Obtain the document frequency. * * @return document frequency **/ - virtual double getDocFreq() const { return _docFreq; } + double getDocFreq() const override final { return _docFreq; } /** * Obtain the match handle for this field. diff --git a/searchlib/src/vespa/searchlib/index/OWNERS b/searchlib/src/vespa/searchlib/index/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchlib/src/vespa/searchlib/index/OWNERS +++ b/searchlib/src/vespa/searchlib/index/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index b9e5bf5a4ea..ba238d0eaeb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_memoryindex OBJECT +vespa_add_library(searchlib_memoryindex OBJECT SOURCES compact_document_words_store.cpp dictionary.cpp diff --git a/searchlib/src/vespa/searchlib/memoryindex/OWNERS b/searchlib/src/vespa/searchlib/memoryindex/OWNERS index e6340232840..b7b549c6058 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/OWNERS +++ b/searchlib/src/vespa/searchlib/memoryindex/OWNERS @@ -1,2 +1,2 @@ -tegge +toregge geirst diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp index 98bf03c1e7c..a0051855c6a 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.cpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.cpp @@ -14,7 +14,7 @@ bool log_enabled() { } void log_debug(vespalib::string &str) { - LOG(debug, str.c_str()); + LOG(debug, "%s", str.c_str()); } } // namespace simpleindex diff --git a/searchlib/src/vespa/searchlib/query/CMakeLists.txt b/searchlib/src/vespa/searchlib/query/CMakeLists.txt index 50aca60fc1c..d255732835e 100644 --- a/searchlib/src/vespa/searchlib/query/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/query/CMakeLists.txt @@ -1,12 +1,10 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_query +vespa_add_library(searchlib_query OBJECT SOURCES queryterm.cpp querynode.cpp base.cpp query.cpp querynoderesultbase.cpp - $<TARGET_OBJECTS:searchlib_tree> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt b/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt index 3f7f5bdb3af..ef1af0cafcb 100644 --- a/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_tree OBJECT +vespa_add_library(searchlib_query_tree OBJECT SOURCES intermediate.cpp intermediatenodes.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 56c77ed46cf..8e6894cc593 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_queryeval +vespa_add_library(searchlib_queryeval OBJECT SOURCES andnotsearch.cpp andsearch.cpp @@ -48,7 +48,5 @@ vespa_add_library(searchlib_queryeval unpackinfo.cpp weighted_set_term_blueprint.cpp weighted_set_term_search.cpp - $<TARGET_OBJECTS:searchlib_queryeval_wand> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 6b23f41a34a..1e0bcb67da6 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(searchlib_test statestring.cpp initrange.cpp document_weight_attribute_helper.cpp - $<TARGET_OBJECTS:searchlib_fakedata> + $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> DEPENDS searchlib_searchlib_test_memoryindex diff --git a/searchlib/src/vespa/searchlib/test/OWNERS b/searchlib/src/vespa/searchlib/test/OWNERS index 64735d11d93..1708c0d4695 100644 --- a/searchlib/src/vespa/searchlib/test/OWNERS +++ b/searchlib/src/vespa/searchlib/test/OWNERS @@ -1 +1 @@ -tegge +toregge diff --git a/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt index b01ad63e02f..eb6bc335957 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_fakedata OBJECT +vespa_add_library(searchlib_test_fakedata OBJECT SOURCES fakeword.cpp fakewordset.cpp diff --git a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt index ea6b238e8b3..15fcda6730f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_translog OBJECT +vespa_add_library(searchlib_transactionlog OBJECT SOURCES common.cpp domain.cpp diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index be1de99efef..9faa80a76b9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -38,7 +38,6 @@ Domain::Domain(const string &domainName, const FileHeaderContext &fileHeaderContext) : _defaultCrcType(defaultCrcType), _executor(executor), - _count(0), _sessionId(1), _useFsync(useFsync), _syncMonitor(), @@ -83,7 +82,6 @@ void Domain::addPart(int64_t partId, bool isLastPart) { } else { { LockGuard guard(_lock); - _count += dp->size(); _parts[partId] = dp; } if (! isLastPart) { @@ -119,7 +117,7 @@ DomainInfo Domain::getDomainInfo() const { LockGuard guard(_lock); - DomainInfo info(SerialNumRange(begin(), end()), count(), byteSize()); + DomainInfo info(SerialNumRange(begin(), end()), size(guard), byteSize()); for (const auto &entry: _parts) { const DomainPart &part = *entry.second; info.parts.emplace_back(PartInfo(part.range(), part.size(), @@ -272,12 +270,8 @@ void Domain::commit(const Packet & packet) } dp = _parts.rbegin()->second; } - size_t oldSz(dp->size()); dp->commit(entry.serial(), packet); cleanSessions(); - // If commit fails no updates should be sent to subscribers either. - // Is is better to keep a consistent behaviour. - _count += dp->size() - oldSz; LockGuard guard(_sessionLock); for (auto & it : _sessions) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 6309d7113f1..d6dbde6b06e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -10,25 +10,25 @@ namespace transactionlog { struct PartInfo { SerialNumRange range; - size_t count; + size_t numEntries; size_t byteSize; vespalib::string file; - PartInfo(SerialNumRange range_in, size_t count_in, + PartInfo(SerialNumRange range_in, size_t numEntries_in, size_t byteSize_in, vespalib::stringref file_in) - : range(range_in), count(count_in), byteSize(byteSize_in), + : range(range_in), numEntries(numEntries_in), byteSize(byteSize_in), file(file_in) {} }; struct DomainInfo { SerialNumRange range; - size_t count; + size_t numEntries; size_t byteSize; std::vector<PartInfo> parts; - DomainInfo(SerialNumRange range_in, size_t count_in, size_t byteSize_in) - : range(range_in), count(count_in), byteSize(byteSize_in), parts() {} + DomainInfo(SerialNumRange range_in, size_t numEntries_in, size_t byteSize_in) + : range(range_in), numEntries(numEntries_in), byteSize(byteSize_in), parts() {} DomainInfo() - : range(), count(0), byteSize(0), parts() {} + : range(), numEntries(0), byteSize(0), parts() {} }; typedef std::map<vespalib::string, DomainInfo> DomainStats; @@ -69,7 +69,6 @@ public: bool getMarkedDeleted(void) const { return _markedDeleted; } void markDeleted(void) { _markedDeleted = true; } - uint64_t count() const { return _count; } size_t byteSize() const; size_t getNumSessions() const { return _sessions.size(); } @@ -103,7 +102,6 @@ private: DomainPart::Crc _defaultCrcType; Executor & _executor; - uint64_t _count; int _sessionId; const bool _useFsync; vespalib::Monitor _syncMonitor; diff --git a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp index 49c16940be5..fdc771a7b58 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.cpp @@ -21,7 +21,7 @@ struct DomainExplorer : vespalib::StateExplorer { DomainInfo info = domain->getDomainInfo(); state.setLong("from", info.range.from()); state.setLong("to", info.range.to()); - state.setLong("count", info.count); + state.setLong("numEntries", info.numEntries); state.setLong("byteSize", info.byteSize); if (full) { Cursor &array = state.setArray("parts"); @@ -29,7 +29,7 @@ struct DomainExplorer : vespalib::StateExplorer { Cursor &part = array.addObject(); part.setLong("from", part_in.range.from()); part.setLong("to", part_in.range.to()); - part.setLong("count", part_in.count); + part.setLong("numEntries", part_in.numEntries); part.setLong("byteSize", part_in.byteSize); part.setString("file", part_in.file); { diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 79b7413c1b4..b9a9ea57a9e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -219,7 +219,6 @@ void TransLogServer::logMetric() const EV_COUNT((prefix + "last").c_str(), it->second->end()); EV_COUNT((prefix + "first").c_str(), it->second->begin()); EV_VALUE((prefix + "numused").c_str(), it->second->size()); - EV_COUNT((prefix + "count").c_str(), it->second->count()); } } diff --git a/searchsummary/.gitignore b/searchsummary/.gitignore index be0452bed21..4b160dd6a2e 100644 --- a/searchsummary/.gitignore +++ b/searchsummary/.gitignore @@ -1,4 +1,3 @@ /target /pom.xml.build Makefile -Testing diff --git a/searchsummary/pom.xml b/searchsummary/pom.xml index 0676ac78527..461a808d232 100644 --- a/searchsummary/pom.xml +++ b/searchsummary/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchsummary</artifactId> <version>6-SNAPSHOT</version> diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 89cb8a2b719..960cbd3605b 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>service-monitor</artifactId> <packaging>container-plugin</packaging> diff --git a/serviceview/pom.xml b/serviceview/pom.xml index bdca519f2b6..8ec82ca2bd2 100644 --- a/serviceview/pom.xml +++ b/serviceview/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>serviceview</artifactId> <packaging>container-plugin</packaging> diff --git a/simplemetrics/pom.xml b/simplemetrics/pom.xml index 2ce63747ff5..7f3388c08ca 100644 --- a/simplemetrics/pom.xml +++ b/simplemetrics/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>simplemetrics</artifactId> <packaging>container-plugin</packaging> diff --git a/slobrok/.gitignore b/slobrok/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/slobrok/.gitignore +++ b/slobrok/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/slobrok/src/apps/slobrok/CMakeLists.txt b/slobrok/src/apps/slobrok/CMakeLists.txt index 1a735a366e4..edb3976b74e 100644 --- a/slobrok/src/apps/slobrok/CMakeLists.txt +++ b/slobrok/src/apps/slobrok/CMakeLists.txt @@ -6,5 +6,4 @@ vespa_add_executable(slobrok_app INSTALL bin DEPENDS slobrok_slobrokserver - slobrok ) diff --git a/slobrok/src/tests/backoff/CMakeLists.txt b/slobrok/src/tests/backoff/CMakeLists.txt index b64b9f36cba..60ec9071f4e 100644 --- a/slobrok/src/tests/backoff/CMakeLists.txt +++ b/slobrok/src/tests/backoff/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(slobrok_backoff_test_app TEST SOURCES testbackoff.cpp DEPENDS - slobrok slobrok_slobrokserver ) vespa_add_test(NAME slobrok_backoff_test_app COMMAND slobrok_backoff_test_app) diff --git a/slobrok/src/tests/configure/CMakeLists.txt b/slobrok/src/tests/configure/CMakeLists.txt index 93718d305b3..86c310c5338 100644 --- a/slobrok/src/tests/configure/CMakeLists.txt +++ b/slobrok/src/tests/configure/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_executable(slobrok_configure_test_app TEST SOURCES configure.cpp DEPENDS - slobrok slobrok_slobrokserver ) vespa_add_test( diff --git a/slobrok/src/tests/mirrorapi/CMakeLists.txt b/slobrok/src/tests/mirrorapi/CMakeLists.txt index 62b1d32eac7..567d1398dbe 100644 --- a/slobrok/src/tests/mirrorapi/CMakeLists.txt +++ b/slobrok/src/tests/mirrorapi/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(slobrok_mirrorapi_test_app TEST SOURCES mirrorapi.cpp DEPENDS - slobrok slobrok_slobrokserver ) vespa_add_test(NAME slobrok_mirrorapi_test_app COMMAND slobrok_mirrorapi_test_app) diff --git a/slobrok/src/tests/oldapi/CMakeLists.txt b/slobrok/src/tests/oldapi/CMakeLists.txt index 0616e0e0696..29ec01930ce 100644 --- a/slobrok/src/tests/oldapi/CMakeLists.txt +++ b/slobrok/src/tests/oldapi/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_executable(slobrok_oldapi_test_app TEST old.cpp mirror.cpp DEPENDS - slobrok slobrok_slobrokserver ) vespa_add_test(NAME slobrok_oldapi_test_app COMMAND slobrok_oldapi_test_app) diff --git a/slobrok/src/tests/registerapi/CMakeLists.txt b/slobrok/src/tests/registerapi/CMakeLists.txt index 366a5adda78..60b7d19b7f5 100644 --- a/slobrok/src/tests/registerapi/CMakeLists.txt +++ b/slobrok/src/tests/registerapi/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_executable(slobrok_registerapi_test_app TEST SOURCES registerapi.cpp DEPENDS - slobrok slobrok_slobrokserver ) vespa_add_test(NAME slobrok_registerapi_test_app COMMAND slobrok_registerapi_test_app) diff --git a/slobrok/src/tests/standalone/CMakeLists.txt b/slobrok/src/tests/standalone/CMakeLists.txt index ce9517a67af..faed1f17c02 100644 --- a/slobrok/src/tests/standalone/CMakeLists.txt +++ b/slobrok/src/tests/standalone/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(slobrok_standalone_test_app TEST standalone.cpp DEPENDS slobrok_slobrokserver - slobrok ) vespa_add_test(NAME slobrok_standalone_test_app COMMAND slobrok_standalone_test_app) diff --git a/slobrok/src/vespa/slobrok/server/CMakeLists.txt b/slobrok/src/vespa/slobrok/server/CMakeLists.txt index d20f6850518..da337624ce3 100644 --- a/slobrok/src/vespa/slobrok/server/CMakeLists.txt +++ b/slobrok/src/vespa/slobrok/server/CMakeLists.txt @@ -23,4 +23,5 @@ vespa_add_library(slobrok_slobrokserver metrics_producer.cpp INSTALL lib64 DEPENDS + slobrok ) diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 53d8eb2423a..5f610ae2e45 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -352,7 +352,7 @@ RPCHooks::rpc_wantAdd(FRT_RPCRequest *req) const char *dSpec = args[2]._string._str; FRT_Values &retval = *req->GetReturn(); OkState state = _rpcsrvmanager.addRemReservation(remsb, dName, dSpec); - if (state.failed() > 1) { + if (state.failed()) { req->SetError(FRTE_RPC_METHOD_FAILED, state.errorMsg.c_str()); } retval.AddInt32(state.errorCode); diff --git a/staging_vespalib/.gitignore b/staging_vespalib/.gitignore index 727ed8ec9e8..65e000de4f0 100644 --- a/staging_vespalib/.gitignore +++ b/staging_vespalib/.gitignore @@ -2,4 +2,3 @@ bin doc lib Makefile -Testing diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index b211d196160..538a885b031 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>standalone-container</artifactId> <version>6-SNAPSHOT</version> @@ -85,7 +84,8 @@ config-bundle-jar-with-dependencies.jar, config-model-api-jar-with-dependencies.jar, config-model-jar-with-dependencies.jar, - container-disc-jar-with-dependencies.jar + container-disc-jar-with-dependencies.jar, + vespajlib.jar </discPreInstallBundle> <bundleActivator>com.yahoo.container.standalone.StandaloneContainerActivator</bundleActivator> <jdiscPrivilegedActivator>true</jdiscPrivilegedActivator> diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala index 0a56f9ac19a..c281cd0eabc 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala @@ -16,13 +16,13 @@ class CloudConfigYinstVariablesTest { @Test def test_configserver_parsing { - val parsed = convert("test2-lulf.trondheim.corp.yahoo.com") + val parsed = convert("myhost.mydomain.com") assertThat(parsed.length, is(1)) } @Test def port_can_be_configured { - val parsed = convert("test1-tonyv:123") + val parsed = convert("myhost:123") val port: Int = parsed(0).port.get() assertThat(port, is(123)) } @@ -38,12 +38,12 @@ class CloudConfigYinstVariablesTest { @Test(expected = classOf[IllegalArgumentException]) def missing_port_gives_exception { - convert("test1-tonyv:") + convert("myhost:") } @Test(expected = classOf[IllegalArgumentException]) def non_numeric_port_gives_exception { - convert("test1-tonyv:non-numeric") + convert("myhost:non-numeric") } @Test diff --git a/statistics/pom.xml b/statistics/pom.xml index 09c81dccb71..e00b611581a 100644 --- a/statistics/pom.xml +++ b/statistics/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>statistics</artifactId> <version>6-SNAPSHOT</version> diff --git a/storage/.gitignore b/storage/.gitignore index 3b41f121299..7d50a4f9b88 100644 --- a/storage/.gitignore +++ b/storage/.gitignore @@ -9,4 +9,3 @@ testLogs /target /pom.xml.build Makefile -Testing diff --git a/storage/OWNERS b/storage/OWNERS index 97c35339850..11a58a546b1 100644 --- a/storage/OWNERS +++ b/storage/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/storage/pom.xml b/storage/pom.xml index b4c8e430f04..ffbc30853b6 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>storage</artifactId> <version>6-SNAPSHOT</version> diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 95228966589..2ba195e70e0 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testbucketdb +vespa_add_library(storage_testbucketdb TEST SOURCES initializertest.cpp bucketmanagertest.cpp @@ -8,7 +8,7 @@ vespa_add_library(storage_testbucketdb lockablemaptest.cpp bucketinfotest.cpp distribution_hash_normalizer_test.cpp - DEPENDS AFTER storage_storageconfig + storage_bucketdb ) diff --git a/storage/src/tests/bucketmover/CMakeLists.txt b/storage/src/tests/bucketmover/CMakeLists.txt index 2d02cdc4942..e2929187f62 100644 --- a/storage/src/tests/bucketmover/CMakeLists.txt +++ b/storage/src/tests/bucketmover/CMakeLists.txt @@ -1,9 +1,8 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testbucketmover +vespa_add_library(storage_testbucketmover TEST SOURCES bucketmovertest.cpp htmltabletest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 309308473e1..737815d85c2 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -1,12 +1,12 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testcommon +vespa_add_library(storage_testcommon TEST SOURCES dummystoragelink.cpp testhelper.cpp metricstest.cpp storagelinktest.cpp teststorageapp.cpp - DEPENDS AFTER storage_storageconfig + storage_bucketdb ) diff --git a/storage/src/tests/common/hostreporter/CMakeLists.txt b/storage/src/tests/common/hostreporter/CMakeLists.txt index f0cb197c5e2..9e3fabdcb29 100644 --- a/storage/src/tests/common/hostreporter/CMakeLists.txt +++ b/storage/src/tests/common/hostreporter/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testhostreporter +vespa_add_library(storage_testhostreporter TEST SOURCES cpureportertest.cpp memreportertest.cpp @@ -8,7 +8,6 @@ vespa_add_library(storage_testhostreporter diskreportertest.cpp util.cpp hostinfotest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 6c6ba62ba6e..29bb8cb67c1 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testdistributor +vespa_add_library(storage_testdistributor TEST SOURCES mergelimitertest.cpp bucketdatabasetest.cpp @@ -38,7 +38,6 @@ vespa_add_library(storage_testdistributor bucketgctimecalculatortest.cpp nodemaintenancestatstrackertest.cpp distributor_host_info_reporter_test.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt index da78716459f..4d89a4aa548 100644 --- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt @@ -1,8 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testmemory +vespa_add_library(storage_testmemory TEST SOURCES memorystatusviewertest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 734be8e9998..95c4d3a532e 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -1,8 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_teststatus +vespa_add_library(storage_teststatus TEST SOURCES statustest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index c065c3eef5b..db0ac64c183 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testpersistence +vespa_add_library(storage_testpersistence TEST SOURCES processalltest.cpp persistencetestutils.cpp @@ -13,7 +13,6 @@ vespa_add_library(storage_testpersistence bucketownershipnotifiertest.cpp persistencequeuetest.cpp testandsettest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index b1314ca0537..2ccebdd8a96 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testfilestorage +vespa_add_library(storage_testfilestorage TEST SOURCES filestormanagertest.cpp operationabortingtest.cpp @@ -11,7 +11,7 @@ vespa_add_library(storage_testfilestorage filestormodifiedbucketstest.cpp deletebuckettest.cpp singlebucketjointest.cpp - DEPENDS AFTER + storage_bucketdb storage_storageconfig ) diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index 2e327089b4c..dacd37d8ea7 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_teststorageserver +vespa_add_library(storage_teststorageserver TEST SOURCES communicationmanagertest.cpp statemanagertest.cpp @@ -11,7 +11,7 @@ vespa_add_library(storage_teststorageserver priorityconvertertest.cpp statereportertest.cpp changedbucketownershiphandlertest.cpp - DEPENDS AFTER + storage_bucketdb storage_storageconfig ) diff --git a/storage/src/tests/storageutil/CMakeLists.txt b/storage/src/tests/storageutil/CMakeLists.txt index a48895352e8..e49361285d8 100644 --- a/storage/src/tests/storageutil/CMakeLists.txt +++ b/storage/src/tests/storageutil/CMakeLists.txt @@ -1,10 +1,9 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_teststorageutil +vespa_add_library(storage_teststorageutil TEST SOURCES functortest.cpp charttest.cpp palettetest.cpp - DEPENDS AFTER storage_storageconfig ) diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt index fde0eb6ac3d..f1c10bde4f0 100644 --- a/storage/src/tests/visiting/CMakeLists.txt +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -1,11 +1,10 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_testvisiting +vespa_add_library(storage_testvisiting TEST SOURCES commandqueuetest.cpp visitormanagertest.cpp visitortest.cpp memory_bounded_trace_test.cpp - DEPENDS AFTER storage_storageconfig storage_visitor diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 74dac0c016c..bc78f3786f6 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -507,7 +507,7 @@ StorageBucketDBInitializer::onDown( std::ostringstream ost; ost << "Cannot perform operation " << msg->getType() << " now because " << "we are still listing buckets from disk."; - LOGBP(warning, ost.str().c_str()); + LOGBP(warning, "%s", ost.str().c_str()); std::unique_ptr<api::StorageReply> reply( static_cast<api::StorageCommand&>(*msg).makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, ost.str())); diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index 3a48573c4ce..23fbf477fd2 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -163,7 +163,7 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) << vespalib::getStackTrace(0); if (!msg->getType().isReply()) { //if (!_closed) { - LOGBP(warning, ost.str().c_str()); + LOGBP(warning, "%s", ost.str().c_str()); //} StorageCommand& cmd = static_cast<StorageCommand&>(*msg); shared_ptr<StorageReply> reply(cmd.makeReply().release()); @@ -177,7 +177,7 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); //if (!_closed) { - LOGBP(warning, ost.str().c_str()); + LOGBP(warning, "%s", ost.str().c_str()); //} } //} @@ -225,7 +225,7 @@ void StorageLink::sendUp(const shared_ptr<StorageMessage> & msg) ost << vespalib::getStackTrace(0); if (!msg->getType().isReply()) { //if (!_closed) { - LOGBP(warning, ost.str().c_str()); + LOGBP(warning, "%s", ost.str().c_str()); //} StorageCommand& cmd = static_cast<StorageCommand&>(*msg); shared_ptr<StorageReply> reply(cmd.makeReply().release()); @@ -239,7 +239,7 @@ void StorageLink::sendUp(const shared_ptr<StorageMessage> & msg) ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); //if (!_closed) { - LOGBP(warning, ost.str().c_str()); + LOGBP(warning, "%s", ost.str().c_str()); //} } //} diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index ee10584ce69..50f9a37e55e 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -171,7 +171,7 @@ namespace { } else if (state != DeadLockDetector::OK) { vespalib::asciistream ost; ost << "Thread " << id << " has registered tick again.\n"; - LOGBT(info, "%s", ost.str().c_str()); + LOGBP(info, "%s", ost.str().c_str()); state = DeadLockDetector::OK; } } diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index bd5b4febf18..3e0aa0fe4b9 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -987,7 +987,7 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, if (_env._fileStorHandler.isMerging(id)) { const char* err = "A merge is already running on this bucket."; - LOG(debug, err); + LOG(debug, "%s", err); tracker->fail(ReturnCode::BUSY, err); return tracker; } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 21c1e68cbcc..a2eb5bf5edd 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -498,7 +498,7 @@ CommunicationManager::enqueue(const std::shared_ptr<api::StorageMessage> & msg) ost << "Failed to aquire " << (memoryFootprint * 2) << " bytes of memory to handle command of type " << msg->getType() << "\n"; - LOG(spam, ost.str().c_str()); + LOG(spam, "%s", ost.str().c_str()); api::StorageCommand* cmd(dynamic_cast<api::StorageCommand*>(msg.get())); if (cmd) { diff --git a/storageapi/.gitignore b/storageapi/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/storageapi/.gitignore +++ b/storageapi/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/storageapi/OWNERS b/storageapi/OWNERS index 97c35339850..11a58a546b1 100644 --- a/storageapi/OWNERS +++ b/storageapi/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/storageframework/.gitignore b/storageframework/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/storageframework/.gitignore +++ b/storageframework/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/storageframework/OWNERS b/storageframework/OWNERS index 97c35339850..11a58a546b1 100644 --- a/storageframework/OWNERS +++ b/storageframework/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/storageserver/.gitignore b/storageserver/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/storageserver/.gitignore +++ b/storageserver/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/storageserver/OWNERS b/storageserver/OWNERS index 97c35339850..11a58a546b1 100644 --- a/storageserver/OWNERS +++ b/storageserver/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/streamingvisitors/.gitignore b/streamingvisitors/.gitignore index 31d449f16ee..ad59e5b91d2 100644 --- a/streamingvisitors/.gitignore +++ b/streamingvisitors/.gitignore @@ -1,3 +1,2 @@ testrun Makefile -Testing diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 6623f743d3b..167fbb306dd 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -12,7 +12,7 @@ vespa_define_module( vdslib vsm - EXTERNAL_DEPENDS + TEST_EXTERNAL_DEPENDS cppunit LIBS diff --git a/testutil/pom.xml b/testutil/pom.xml index cd20d0db597..0d2aa8a4dbf 100644 --- a/testutil/pom.xml +++ b/testutil/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>testutil</artifactId> <packaging>jar</packaging> diff --git a/vbench/.gitignore b/vbench/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/vbench/.gitignore +++ b/vbench/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/vdslib/.gitignore b/vdslib/.gitignore index 96c62777acb..b0a64672599 100644 --- a/vdslib/.gitignore +++ b/vdslib/.gitignore @@ -2,4 +2,3 @@ target vdslib-lib.iml /pom.xml.build Makefile -Testing diff --git a/vdslib/OWNERS b/vdslib/OWNERS index 97c35339850..11a58a546b1 100644 --- a/vdslib/OWNERS +++ b/vdslib/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/vdslib/pom.xml b/vdslib/pom.xml index 994fe8ddf5e..0f769fe5b3f 100644 --- a/vdslib/pom.xml +++ b/vdslib/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vdslib</artifactId> <packaging>container-plugin</packaging> diff --git a/vdstestlib/.gitignore b/vdstestlib/.gitignore index a9b20e8992d..f3c7a7c5da6 100644 --- a/vdstestlib/.gitignore +++ b/vdstestlib/.gitignore @@ -1,2 +1 @@ Makefile -Testing diff --git a/vdstestlib/OWNERS b/vdstestlib/OWNERS index 97c35339850..11a58a546b1 100644 --- a/vdstestlib/OWNERS +++ b/vdstestlib/OWNERS @@ -1,2 +1,2 @@ vekterli -dybdahl +dybis diff --git a/vespa-application-maven-plugin/pom.xml b/vespa-application-maven-plugin/pom.xml index 1ad7a335d47..a70525e67de 100644 --- a/vespa-application-maven-plugin/pom.xml +++ b/vespa-application-maven-plugin/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-application-maven-plugin</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespa-documentgen-plugin/OWNERS b/vespa-documentgen-plugin/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/vespa-documentgen-plugin/OWNERS +++ b/vespa-documentgen-plugin/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index f75637e1978..0c5b957c14b 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-documentgen-plugin</artifactId> <packaging>maven-plugin</packaging> diff --git a/vespa-http-client/OWNERS b/vespa-http-client/OWNERS index 69bebdee173..efd3ad88b9a 100644 --- a/vespa-http-client/OWNERS +++ b/vespa-http-client/OWNERS @@ -1,2 +1,2 @@ -dybdahl +dybis vekterli diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index b4ae744734e..ad352cfe4cd 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-http-client</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java index f45e1a2edac..8f943ea288a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java @@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicInteger; * * A {@link FeedClientFactory} is provided to instantiate Sessions. * - * @author dybdahl + * @author dybis * @see FeedClientFactory * @since 5.29.0 */ diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java index 39356eb33c6..262cb3e1ca4 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java @@ -9,7 +9,7 @@ import static com.yahoo.vespa.http.client.SessionFactory.createTimeoutExecutor; /** * Factory for creating FeedClient. - * @author dybdahl + * @author dybis */ public class FeedClientFactory { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java index 17d1313336e..391c90003e7 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java @@ -13,7 +13,7 @@ import java.util.concurrent.atomic.AtomicInteger; * "Result received: 34 (1 failed so far, 2003 sent, success rate 1999.23 docs/sec)." * On each failure it will print the Result object content. If tracing is enabled, it will print trace messages to * std err as well. - * @author dybdahl + * @author dybis */ public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java index 6e25082a7a0..02a9d781840 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.http.client.Result; /** * Result from a single endpoint. - * @author dybdahl + * @author dybis */ public class EndpointResult { private final String operationId; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java index 530e22f8abc..31fca0a0d3d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java @@ -13,7 +13,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Reads a stream of json documents and sends them to feedClient. - * @author dybdahl + * @author dybis */ public class JsonReader { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java index 6c24f12c84c..f739964211a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java @@ -7,7 +7,7 @@ import static java.lang.Math.min; /** * Class that has a method for finding next maxInFlight. - * @author dybdahl + * @author dybis */ public class ThrottlePolicy { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java index 3f1f896ff23..e22f16a200c 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java @@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Reads an input stream of xml, sends these to session. - * @author dybdahl + * @author dybis */ public class XmlFeedReader { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java index 87f83d56840..62a93c84f79 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java @@ -15,7 +15,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; /** * Implementation of FeedClient. It is a thin layer on top of multiClusterHandler and multiClusterResultAggregator. - * @author dybdahl + * @author dybis */ public class FeedClientImpl implements FeedClient { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java index cfe63610f4b..2f696d69e32 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java @@ -9,7 +9,7 @@ import java.io.IOException; /** * Class for wiring up the Session API. It is the return value of stream() in the Session API. - * @author dybdahl + * @author dybis */ class MultiClusterSessionOutputStream extends ByteArrayOutputStream { private final CharSequence documentId; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java index 0e6d6575dc6..6462ee1b2cd 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java @@ -11,7 +11,7 @@ import java.util.concurrent.TimeUnit; /** * Document queue that only gives you document operations on documents for which there are no already in flight operations for. - * @author dybdahl + * @author dybis */ class DocumentQueue { private final Deque<Document> queue; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java index 4fc91e1eb0e..f634a2e22f7 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java @@ -17,7 +17,7 @@ import java.util.List; /** * Dummy implementation. * - * @author dybdahl + * @author dybis */ public class DryRunGatewayConnection implements GatewayConnection { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java index 1bb4d2ffeb0..e0256b33c60 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java @@ -7,7 +7,7 @@ import java.io.IOException; /** * Class for throwing exception from endpoint. - * @author dybdahl + * @author dybis */ public class EndpointIOException extends IOException { private final Endpoint endpoint; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java index e26beee6890..4041ec21c44 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java @@ -7,7 +7,7 @@ import java.util.Random; * When the gateways says it can not handle more load, we should send less load. That is the responsibility * of this component * - * @author dybdahl + * @author dybis */ public class GatewayThrottler { private long backOffTimeMs = 0; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java index 6072ad7f87b..48d58dcb354 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java @@ -5,7 +5,7 @@ import java.util.concurrent.Semaphore; /** * A semaphore that can be re-sized. - * @author dybdahl + * @author dybis */ final public class ConcurrentDocumentOperationBlocker { @@ -64,4 +64,4 @@ final public class ConcurrentDocumentOperationBlocker { super.reducePermits(reduction); } } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java index 604617bda36..342b197c7b3 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java @@ -27,7 +27,7 @@ import java.util.Random; * * Class is fully thread safe, i.e. all public methods are thread safe. * - * @author dybdahl + * @author dybis */ public class IncompleteResultsThrottler { private final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 1d712405795..b736a727c8f 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -29,7 +29,7 @@ import java.util.logging.Logger; /** * Merges several endpointResult into one Result and does the callback. - * @author dybdahl + * @author dybis * @since 5.1.20 */ @Beta diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 22f871ffbf8..80a24bc17f2 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -19,7 +19,7 @@ import java.util.concurrent.TimeUnit; /** * Commandline interface for the binary. - * @author dybdahl + * @author dybis */ @Beta @Command(name = "vespa-http-client", diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index 1a10be995c5..bd0b8d5276f 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @author dybdahl + * @author dybis * @since 5.1.20 */ public class Runner { diff --git a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java index 3af5e1d2435..923e4ed83ff 100644 --- a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java +++ b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java @@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Unit test that test documentation code. - * @author dybdahl + * @author dybis */ public class ExampleUsageFeedClientTest { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java index 872bc4192b2..0e7b281db26 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java @@ -20,7 +20,7 @@ import static org.junit.Assert.*; /** * Tests for the API, using dryrun option to mock gateway. - * @author dybdahl + * @author dybis */ public class FeedClientTest { @@ -81,4 +81,4 @@ public class FeedClientTest { feedClient.close(); assertThat(resultsReceived.get(), is(1)); } -}
\ No newline at end of file +} diff --git a/vespa_application_maven_archetype_plugin/pom.xml b/vespa_application_maven_archetype_plugin/pom.xml index e1c2fb8e89a..4f9c8ae444d 100644 --- a/vespa_application_maven_archetype_plugin/pom.xml +++ b/vespa_application_maven_archetype_plugin/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <!-- Package name must contain _maven_archetype --> <artifactId>vespa_application_maven_archetype_plugin</artifactId> diff --git a/vespa_feed_perf/OWNERS b/vespa_feed_perf/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/vespa_feed_perf/OWNERS +++ b/vespa_feed_perf/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index c78d18f055a..35f292d5f16 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -13,7 +13,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa_feed_perf</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespa_jersey2/pom.xml b/vespa_jersey2/pom.xml index d510e1f92d5..5d65eadec40 100644 --- a/vespa_jersey2/pom.xml +++ b/vespa_jersey2/pom.xml @@ -9,7 +9,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa_jersey2</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespaapplication/OWNERS b/vespaapplication/OWNERS deleted file mode 100644 index 338ed581212..00000000000 --- a/vespaapplication/OWNERS +++ /dev/null @@ -1 +0,0 @@ -hmusum diff --git a/vespaapplication/README b/vespaapplication/README deleted file mode 100644 index 13cfeff9208..00000000000 --- a/vespaapplication/README +++ /dev/null @@ -1 +0,0 @@ -This package contains Vespa sample applications. diff --git a/vespaapplication/examples/music-docproc/docproc/ExampleDocumentProcessor.java b/vespaapplication/examples/music-docproc/docproc/ExampleDocumentProcessor.java deleted file mode 100644 index e5386b9ad66..00000000000 --- a/vespaapplication/examples/music-docproc/docproc/ExampleDocumentProcessor.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.document.Document; -import com.yahoo.docproc.*; - -public class ExampleDocumentProcessor extends DocumentProcessor { - - public Progress process(Document document, Arguments arguments, Processing processing) { - document.setValue("title", "Worst music ever"); - System.out.println("Processed " + document); - return Progress.DONE; - } - -} diff --git a/vespaapplication/examples/music-docproc/docproc/README b/vespaapplication/examples/music-docproc/docproc/README deleted file mode 100644 index e19a99e6aa3..00000000000 --- a/vespaapplication/examples/music-docproc/docproc/README +++ /dev/null @@ -1 +0,0 @@ -Included here is ExampleDocumentProcessor.java from Vespa documentation. diff --git a/vespaapplication/examples/music-docproc/hosts.xml b/vespaapplication/examples/music-docproc/hosts.xml deleted file mode 100644 index 23287fe54f2..00000000000 --- a/vespaapplication/examples/music-docproc/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/vespaapplication/examples/music-docproc/searchdefinitions/music.sd b/vespaapplication/examples/music-docproc/searchdefinitions/music.sd deleted file mode 100644 index 9e895547cb1..00000000000 --- a/vespaapplication/examples/music-docproc/searchdefinitions/music.sd +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# A basic search definition - called music, should be saved to music.sd -search music { - - # It contains one document type only - called music as well - document music { - - field title type string { - indexing: summary | index # How this field should be indexed - weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature - header - } - - field artist type string { - indexing: summary | attribute | index - weight: 25 - header - } - - field year type int { - indexing: summary | attribute - header - } - - # Increase query - field popularity type int { - indexing: summary | attribute - header - } - - field url type uri { - indexing: summary | index - header - } - - field cover type raw { - body - } - - } - - fieldset default { - fields: title, artist - } - - rank-profile default inherits default { - first-phase { - expression: nativeRank(title,artist) + attribute(popularity) - } - - } - - rank-profile textmatch inherits default { - first-phase { - expression: nativeRank(title,artist) - } - - } - - - -} diff --git a/vespaapplication/examples/music-docproc/services.xml b/vespaapplication/examples/music-docproc/services.xml deleted file mode 100644 index 82139c1562d..00000000000 --- a/vespaapplication/examples/music-docproc/services.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - <slobroks> - <slobrok hostalias="node1" /> - </slobroks> - </admin> - - <jdisc version="1.0"> - <docproc> - <chains> - <chain id="default"> - <documentprocessor id="ExampleDocumentProcessor"/> - </chain> - </chains> - </docproc> - - <search/> - - <document-api/> - - <nodes> - <node hostalias="node1" /> - </nodes> - </jdisc> - - <content version="1.0" id="search"> - <redundancy>1</redundancy> - - <documents> - <document type="music" mode="index"/> - </documents> - - <nodes> - <node hostalias="node1" distribution-key="0"/> - </nodes> - </content> - -</services> diff --git a/vespaapplication/examples/music/hosts.xml b/vespaapplication/examples/music/hosts.xml deleted file mode 100644 index 3ab86a21aef..00000000000 --- a/vespaapplication/examples/music/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/vespaapplication/examples/music/query-profiles/default.xml b/vespaapplication/examples/music/query-profiles/default.xml deleted file mode 100644 index 00742e468f7..00000000000 --- a/vespaapplication/examples/music/query-profiles/default.xml +++ /dev/null @@ -1,4 +0,0 @@ -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<query-profile id="default"> - <field name="presentation.format">JsonRenderer</field> -</query-profile> diff --git a/vespaapplication/examples/music/searchdefinitions/music.sd b/vespaapplication/examples/music/searchdefinitions/music.sd deleted file mode 100644 index 9e895547cb1..00000000000 --- a/vespaapplication/examples/music/searchdefinitions/music.sd +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# A basic search definition - called music, should be saved to music.sd -search music { - - # It contains one document type only - called music as well - document music { - - field title type string { - indexing: summary | index # How this field should be indexed - weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature - header - } - - field artist type string { - indexing: summary | attribute | index - weight: 25 - header - } - - field year type int { - indexing: summary | attribute - header - } - - # Increase query - field popularity type int { - indexing: summary | attribute - header - } - - field url type uri { - indexing: summary | index - header - } - - field cover type raw { - body - } - - } - - fieldset default { - fields: title, artist - } - - rank-profile default inherits default { - first-phase { - expression: nativeRank(title,artist) + attribute(popularity) - } - - } - - rank-profile textmatch inherits default { - first-phase { - expression: nativeRank(title,artist) - } - - } - - - -} diff --git a/vespaapplication/examples/musicdata-extended.xml b/vespaapplication/examples/musicdata-extended.xml deleted file mode 100644 index 52dc093f301..00000000000 --- a/vespaapplication/examples/musicdata-extended.xml +++ /dev/null @@ -1,6052 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/a-ha/Scoundrel+Days"> - <url>http://music.yahoo.com/a-ha/Scoundrel+Days</url> - <title><![CDATA[Scoundrel Days]]></title> - <artist><![CDATA[a-ha]]></artist> - <year>0</year> - <popularity>290</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Accept/Restless+And+Wild"> - <url>http://music.yahoo.com/Accept/Restless+And+Wild</url> - <title><![CDATA[Restless And Wild]]></title> - <artist><![CDATA[Accept]]></artist> - <year>0</year> - <popularity>75</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Accept/Staying+A+Life"> - <url>http://music.yahoo.com/Accept/Staying+A+Life</url> - <title><![CDATA[Staying A Life]]></title> - <artist><![CDATA[Accept]]></artist> - <year>1985</year> - <popularity>77</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Alice+In+Chains/Dirt"> - <url>http://music.yahoo.com/Alice+In+Chains/Dirt</url> - <title><![CDATA[Dirt]]></title> - <artist><![CDATA[Alice In Chains]]></artist> - <year>1992</year> - <popularity>114</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Alice+In+Chains/Live"> - <url>http://music.yahoo.com/Alice+In+Chains/Live</url> - <title><![CDATA[Live]]></title> - <artist><![CDATA[Alice In Chains]]></artist> - <year>1990</year> - <popularity>363</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Amy+MacDonald/This+Is+The+Life"> - <url>http://music.yahoo.com/Amy+MacDonald/This+Is+The+Life</url> - <title><![CDATA[This Is The Life]]></title> - <artist><![CDATA[Amy MacDonald]]></artist> - <year>2007</year> - <popularity>355</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ane+Brun/Duets"> - <url>http://music.yahoo.com/Ane+Brun/Duets</url> - <title><![CDATA[Duets]]></title> - <artist><![CDATA[Ane Brun]]></artist> - <year>0</year> - <popularity>255</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Annuals/Be+He+Me"> - <url>http://music.yahoo.com/Annuals/Be+He+Me</url> - <title><![CDATA[Be He Me]]></title> - <artist><![CDATA[Annuals]]></artist> - <year>0</year> - <popularity>207</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Anti-Nowhere+League/Unknown+Album"> - <url>http://music.yahoo.com/Anti-Nowhere+League/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Anti-Nowhere League]]></artist> - <year>0</year> - <popularity>46</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Aqpop/Beautifully+Smart"> - <url>http://music.yahoo.com/Aqpop/Beautifully+Smart</url> - <title><![CDATA[Beautifully Smart]]></title> - <artist><![CDATA[Aqpop]]></artist> - <year>2004</year> - <popularity>479</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Asobi+Seksu/Citrus"> - <url>http://music.yahoo.com/Asobi+Seksu/Citrus</url> - <title><![CDATA[Citrus]]></title> - <artist><![CDATA[Asobi Seksu]]></artist> - <year>0</year> - <popularity>383</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Asta+Kask/Unknown+Album"> - <url>http://music.yahoo.com/Asta+Kask/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Asta Kask]]></artist> - <year>0</year> - <popularity>115</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Astroburger/I+Used+To+Be+Mod"> - <url>http://music.yahoo.com/Astroburger/I+Used+To+Be+Mod</url> - <title><![CDATA[I Used To Be Mod]]></title> - <artist><![CDATA[Astroburger]]></artist> - <year>0</year> - <popularity>486</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Astroburger/Stand+On+It"> - <url>http://music.yahoo.com/Astroburger/Stand+On+It</url> - <title><![CDATA[Stand On It]]></title> - <artist><![CDATA[Astroburger]]></artist> - <year>0</year> - <popularity>246</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Autopsy/Mental+Funeral"> - <url>http://music.yahoo.com/Autopsy/Mental+Funeral</url> - <title><![CDATA[Mental Funeral]]></title> - <artist><![CDATA[Autopsy]]></artist> - <year>0</year> - <popularity>420</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Backstreet+Boys/Backstreet+Boys"> - <url>http://music.yahoo.com/Backstreet+Boys/Backstreet+Boys</url> - <title><![CDATA[Backstreet Boys]]></title> - <artist><![CDATA[Backstreet Boys]]></artist> - <year>0</year> - <popularity>94</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Beirut/Gulag+Orkestar"> - <url>http://music.yahoo.com/Beirut/Gulag+Orkestar</url> - <title><![CDATA[Gulag Orkestar]]></title> - <artist><![CDATA[Beirut]]></artist> - <year>0</year> - <popularity>380</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bettie+Serveert/Lamprey"> - <url>http://music.yahoo.com/Bettie+Serveert/Lamprey</url> - <title><![CDATA[Lamprey]]></title> - <artist><![CDATA[Bettie Serveert]]></artist> - <year>1995</year> - <popularity>475</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Billy+Idol/Greatest+Hits"> - <url>http://music.yahoo.com/Billy+Idol/Greatest+Hits</url> - <title><![CDATA[Greatest Hits]]></title> - <artist><![CDATA[Billy Idol]]></artist> - <year>0</year> - <popularity>273</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Debbath/Black+Debbath+hyller+kvinnen%21"> - <url>http://music.yahoo.com/Black+Debbath/Black+Debbath+hyller+kvinnen%21</url> - <title><![CDATA[Black Debbath hyller kvinnen!]]></title> - <artist><![CDATA[Black Debbath]]></artist> - <year>2007</year> - <popularity>386</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Debbath/Metal+Tribute+To+Led+Zeppelin"> - <url>http://music.yahoo.com/Black+Debbath/Metal+Tribute+To+Led+Zeppelin</url> - <title><![CDATA[Metal Tribute To Led Zeppelin]]></title> - <artist><![CDATA[Black Debbath]]></artist> - <year>0</year> - <popularity>421</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Debbath/Moto%CC%88rhedda+Gabler"> - <url>http://music.yahoo.com/Black+Debbath/Moto%CC%88rhedda+Gabler</url> - <title><![CDATA[Motörhedda Gabler]]></title> - <artist><![CDATA[Black Debbath]]></artist> - <year>0</year> - <popularity>149</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Debbath/Naar+vi+d%C3%B8de+rocker+%28En+tungrockhyllest+til+Ibsen%29"> - <url>http://music.yahoo.com/Black+Debbath/Naar+vi+d%C3%B8de+rocker+%28En+tungrockhyllest+til+Ibsen%29</url> - <title><![CDATA[Naar vi døde rocker (En tungrockhyllest til Ibsen)]]></title> - <artist><![CDATA[Black Debbath]]></artist> - <year>2006</year> - <popularity>72</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Sabbath/Unknown+Album"> - <url>http://music.yahoo.com/Black+Sabbath/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Black Sabbath]]></artist> - <year>0</year> - <popularity>366</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Blitzkrieg/Unknown+Album"> - <url>http://music.yahoo.com/Blitzkrieg/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Blitzkrieg]]></artist> - <year>0</year> - <popularity>201</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Blue+Oyster+Cult/Unknown+Album"> - <url>http://music.yahoo.com/Blue+Oyster+Cult/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Blue Oyster Cult]]></artist> - <year>0</year> - <popularity>293</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Blue+O%CC%88yster+Cult/Cult+Classic"> - <url>http://music.yahoo.com/Blue+O%CC%88yster+Cult/Cult+Classic</url> - <title><![CDATA[Cult Classic]]></title> - <artist><![CDATA[Blue Öyster Cult]]></artist> - <year>0</year> - <popularity>104</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bo+Kaspers+Orkester/Hund"> - <url>http://music.yahoo.com/Bo+Kaspers+Orkester/Hund</url> - <title><![CDATA[Hund]]></title> - <artist><![CDATA[Bo Kaspers Orkester]]></artist> - <year>0</year> - <popularity>462</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Dylan/Modern+Times"> - <url>http://music.yahoo.com/Bob+Dylan/Modern+Times</url> - <title><![CDATA[Modern Times]]></title> - <artist><![CDATA[Bob Dylan]]></artist> - <year>2006</year> - <popularity>251</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Seger/Unknown+Album"> - <url>http://music.yahoo.com/Bob+Seger/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Bob Seger]]></artist> - <year>0</year> - <popularity>496</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bokklubbens+Barn/Boka+Med+Det+Rare+I"> - <url>http://music.yahoo.com/Bokklubbens+Barn/Boka+Med+Det+Rare+I</url> - <title><![CDATA[Boka Med Det Rare I]]></title> - <artist><![CDATA[Bokklubbens Barn]]></artist> - <year>0</year> - <popularity>247</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bomber/O+Brave+Adventurer"> - <url>http://music.yahoo.com/Bomber/O+Brave+Adventurer</url> - <title><![CDATA[O Brave Adventurer]]></title> - <artist><![CDATA[Bomber]]></artist> - <year>0</year> - <popularity>218</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bomber/The+last+place+on+earth"> - <url>http://music.yahoo.com/Bomber/The+last+place+on+earth</url> - <title><![CDATA[The last place on earth]]></title> - <artist><![CDATA[Bomber]]></artist> - <year>0</year> - <popularity>125</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Book+Of+Black+Earth/Horoskopus"> - <url>http://music.yahoo.com/Book+Of+Black+Earth/Horoskopus</url> - <title><![CDATA[Horoskopus]]></title> - <artist><![CDATA[Book Of Black Earth]]></artist> - <year>2008</year> - <popularity>318</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bruce+Springsteen/Magic"> - <url>http://music.yahoo.com/Bruce+Springsteen/Magic</url> - <title><![CDATA[Magic]]></title> - <artist><![CDATA[Bruce Springsteen]]></artist> - <year>0</year> - <popularity>496</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Budgie/Unknown+Album"> - <url>http://music.yahoo.com/Budgie/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Budgie]]></artist> - <year>0</year> - <popularity>37</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Carl+Orff/Unknown"> - <url>http://music.yahoo.com/Carl+Orff/Unknown</url> - <title><![CDATA[Unknown]]></title> - <artist><![CDATA[Carl Orff]]></artist> - <year>0</year> - <popularity>97</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Celtic+Frost/Monotheist"> - <url>http://music.yahoo.com/Celtic+Frost/Monotheist</url> - <title><![CDATA[Monotheist]]></title> - <artist><![CDATA[Celtic Frost]]></artist> - <year>2006</year> - <popularity>124</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Chris+Rea/Driving+Home+for+Christmas+-+EP"> - <url>http://music.yahoo.com/Chris+Rea/Driving+Home+for+Christmas+-+EP</url> - <title><![CDATA[Driving Home for Christmas - EP]]></title> - <artist><![CDATA[Chris Rea]]></artist> - <year>0</year> - <popularity>30</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Coldplay/X+%26+Y"> - <url>http://music.yahoo.com/Coldplay/X+%26+Y</url> - <title><![CDATA[X & Y]]></title> - <artist><![CDATA[Coldplay]]></artist> - <year>0</year> - <popularity>474</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/1994-Death+Row"> - <url>http://music.yahoo.com/Compilations/1994-Death+Row</url> - <title><![CDATA[1994-Death Row]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1994</year> - <popularity>98</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/All+areas+worldwide"> - <url>http://music.yahoo.com/Compilations/All+areas+worldwide</url> - <title><![CDATA[All areas worldwide]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1997</year> - <popularity>421</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Det+Beste+Til+Meg+Og+Mine+Venner"> - <url>http://music.yahoo.com/Compilations/Det+Beste+Til+Meg+Og+Mine+Venner</url> - <title><![CDATA[Det Beste Til Meg Og Mine Venner]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>2005</year> - <popularity>327</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Ellediller+%26+krokofanter"> - <url>http://music.yahoo.com/Compilations/Ellediller+%26+krokofanter</url> - <title><![CDATA[Ellediller & krokofanter]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1997</year> - <popularity>252</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/EP%27s+Going+Steady"> - <url>http://music.yahoo.com/Compilations/EP%27s+Going+Steady</url> - <title><![CDATA[EP's Going Steady]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1998</year> - <popularity>368</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Kelis+Was+Here"> - <url>http://music.yahoo.com/Compilations/Kelis+Was+Here</url> - <title><![CDATA[Kelis Was Here]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>493</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Kickar"> - <url>http://music.yahoo.com/Compilations/Kickar</url> - <title><![CDATA[Kickar]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1990</year> - <popularity>464</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Larmende+Opptog+i+Taushetsgata"> - <url>http://music.yahoo.com/Compilations/Larmende+Opptog+i+Taushetsgata</url> - <title><![CDATA[Larmende Opptog i Taushetsgata]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1996</year> - <popularity>158</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Objection+Overruled"> - <url>http://music.yahoo.com/Compilations/Objection+Overruled</url> - <title><![CDATA[Objection Overruled]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1993</year> - <popularity>250</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Sp%C3%A6ll+a%CC%8At+m%C3%A6"> - <url>http://music.yahoo.com/Compilations/Sp%C3%A6ll+a%CC%8At+m%C3%A6</url> - <title><![CDATA[Spæll åt mæ]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>2003</year> - <popularity>398</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Starmelt"> - <url>http://music.yahoo.com/Compilations/Starmelt</url> - <title><![CDATA[Starmelt]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1997</year> - <popularity>426</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Hits+_+The+B-Sides+%28Box+Set%29"> - <url>http://music.yahoo.com/Compilations/The+Hits+_+The+B-Sides+%28Box+Set%29</url> - <title><![CDATA[The Hits _ The B-Sides (Box Set)]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1993</year> - <popularity>127</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Trond+Viggos+Beste"> - <url>http://music.yahoo.com/Compilations/Trond+Viggos+Beste</url> - <title><![CDATA[Trond Viggos Beste]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1997</year> - <popularity>330</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/David+Gray/White+Ladder"> - <url>http://music.yahoo.com/David+Gray/White+Ladder</url> - <title><![CDATA[White Ladder]]></title> - <artist><![CDATA[David Gray]]></artist> - <year>0</year> - <popularity>3</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Deep+Purple/Live+In+Japan"> - <url>http://music.yahoo.com/Deep+Purple/Live+In+Japan</url> - <title><![CDATA[Live In Japan]]></title> - <artist><![CDATA[Deep Purple]]></artist> - <year>0</year> - <popularity>18</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Definitivt/Definitivt+Demo"> - <url>http://music.yahoo.com/Definitivt/Definitivt+Demo</url> - <title><![CDATA[Definitivt Demo]]></title> - <artist><![CDATA[Definitivt]]></artist> - <year>2002</year> - <popularity>112</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Black+Celebration"> - <url>http://music.yahoo.com/Depeche+Mode/Black+Celebration</url> - <title><![CDATA[Black Celebration]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1986</year> - <popularity>353</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Diamond+Head/Unknown"> - <url>http://music.yahoo.com/Diamond+Head/Unknown</url> - <title><![CDATA[Unknown]]></title> - <artist><![CDATA[Diamond Head]]></artist> - <year>0</year> - <popularity>493</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Discharge/Unknown+Album"> - <url>http://music.yahoo.com/Discharge/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Discharge]]></artist> - <year>0</year> - <popularity>184</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Disney/Bambi"> - <url>http://music.yahoo.com/Disney/Bambi</url> - <title><![CDATA[Bambi]]></title> - <artist><![CDATA[Disney]]></artist> - <year>2004</year> - <popularity>141</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Disney/Ole+Brumm+og+honningtreet"> - <url>http://music.yahoo.com/Disney/Ole+Brumm+og+honningtreet</url> - <title><![CDATA[Ole Brumm og honningtreet]]></title> - <artist><![CDATA[Disney]]></artist> - <year>2002</year> - <popularity>452</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Diverse/Jul+Pa%CC%8A+Ma%CC%8Anetoppen"> - <url>http://music.yahoo.com/Diverse/Jul+Pa%CC%8A+Ma%CC%8Anetoppen</url> - <title><![CDATA[Jul På Månetoppen]]></title> - <artist><![CDATA[Diverse]]></artist> - <year>0</year> - <popularity>242</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dumdum+Boys/Gravitasjon"> - <url>http://music.yahoo.com/Dumdum+Boys/Gravitasjon</url> - <title><![CDATA[Gravitasjon]]></title> - <artist><![CDATA[Dumdum Boys]]></artist> - <year>2006</year> - <popularity>299</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/D%C3%B8dheimsgard/666+International"> - <url>http://music.yahoo.com/D%C3%B8dheimsgard/666+International</url> - <title><![CDATA[666 International]]></title> - <artist><![CDATA[Dødheimsgard]]></artist> - <year>0</year> - <popularity>94</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ebba+Gro%CC%88n/Ebba+Gro%CC%88n+%28Disc+4%29"> - <url>http://music.yahoo.com/Ebba+Gro%CC%88n/Ebba+Gro%CC%88n+%28Disc+4%29</url> - <title><![CDATA[Ebba Grön (Disc 4)]]></title> - <artist><![CDATA[Ebba Grön]]></artist> - <year>1998</year> - <popularity>183</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ebba+Gro%CC%88n/Ka%CC%88rlek+%26+Uppror"> - <url>http://music.yahoo.com/Ebba+Gro%CC%88n/Ka%CC%88rlek+%26+Uppror</url> - <title><![CDATA[Kärlek & Uppror]]></title> - <artist><![CDATA[Ebba Grön]]></artist> - <year>1981</year> - <popularity>411</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/ElRodeo/Nomansland"> - <url>http://music.yahoo.com/ElRodeo/Nomansland</url> - <title><![CDATA[Nomansland]]></title> - <artist><![CDATA[ElRodeo]]></artist> - <year>0</year> - <popularity>7</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Entombed/Left+Hand+Path"> - <url>http://music.yahoo.com/Entombed/Left+Hand+Path</url> - <title><![CDATA[Left Hand Path]]></title> - <artist><![CDATA[Entombed]]></artist> - <year>1990</year> - <popularity>410</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Erlend+Loe/Doppler+-+lest+av+Erlend+Loe"> - <url>http://music.yahoo.com/Erlend+Loe/Doppler+-+lest+av+Erlend+Loe</url> - <title><![CDATA[Doppler - lest av Erlend Loe]]></title> - <artist><![CDATA[Erlend Loe]]></artist> - <year>0</year> - <popularity>497</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Espen+Sandvik/Tellefanten+-+5"> - <url>http://music.yahoo.com/Espen+Sandvik/Tellefanten+-+5</url> - <title><![CDATA[Tellefanten - 5]]></title> - <artist><![CDATA[Espen Sandvik]]></artist> - <year>2006</year> - <popularity>388</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eurythmics/Revenge+%28Deluxe+Edition%29+%5BRemastered%5D"> - <url>http://music.yahoo.com/Eurythmics/Revenge+%28Deluxe+Edition%29+%5BRemastered%5D</url> - <title><![CDATA[Revenge (Deluxe Edition) [Remastered]]]></title> - <artist><![CDATA[Eurythmics]]></artist> - <year>0</year> - <popularity>406</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eurythmics/Ultimate+Collection"> - <url>http://music.yahoo.com/Eurythmics/Ultimate+Collection</url> - <title><![CDATA[Ultimate Collection]]></title> - <artist><![CDATA[Eurythmics]]></artist> - <year>0</year> - <popularity>303</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Exodus/Shovel+Headed+Kill+Machine"> - <url>http://music.yahoo.com/Exodus/Shovel+Headed+Kill+Machine</url> - <title><![CDATA[Shovel Headed Kill Machine]]></title> - <artist><![CDATA[Exodus]]></artist> - <year>0</year> - <popularity>452</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Exodus/The+Atrocity+Exhibition+-+Exhibit+A"> - <url>http://music.yahoo.com/Exodus/The+Atrocity+Exhibition+-+Exhibit+A</url> - <title><![CDATA[The Atrocity Exhibition - Exhibit A]]></title> - <artist><![CDATA[Exodus]]></artist> - <year>2007</year> - <popularity>442</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Family+Man/Achtung+Baby"> - <url>http://music.yahoo.com/Family+Man/Achtung+Baby</url> - <title><![CDATA[Achtung Baby]]></title> - <artist><![CDATA[Family Man]]></artist> - <year>0</year> - <popularity>178</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Foo+Fighters/The+Colour+And+The+Shape"> - <url>http://music.yahoo.com/Foo+Fighters/The+Colour+And+The+Shape</url> - <title><![CDATA[The Colour And The Shape]]></title> - <artist><![CDATA[Foo Fighters]]></artist> - <year>1997</year> - <popularity>112</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gavin+DeGraw/Chariot"> - <url>http://music.yahoo.com/Gavin+DeGraw/Chariot</url> - <title><![CDATA[Chariot]]></title> - <artist><![CDATA[Gavin DeGraw]]></artist> - <year>0</year> - <popularity>130</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Green+Day/American+Idiot"> - <url>http://music.yahoo.com/Green+Day/American+Idiot</url> - <title><![CDATA[American Idiot]]></title> - <artist><![CDATA[Green Day]]></artist> - <year>0</year> - <popularity>230</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Green+Day/Bullet+In+A+Bible"> - <url>http://music.yahoo.com/Green+Day/Bullet+In+A+Bible</url> - <title><![CDATA[Bullet In A Bible]]></title> - <artist><![CDATA[Green Day]]></artist> - <year>2005</year> - <popularity>287</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Green+Day/Nimrod"> - <url>http://music.yahoo.com/Green+Day/Nimrod</url> - <title><![CDATA[Nimrod]]></title> - <artist><![CDATA[Green Day]]></artist> - <year>0</year> - <popularity>413</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Hanne+Hukkelberg/Aurgasm+Summer+Soundtrack"> - <url>http://music.yahoo.com/Hanne+Hukkelberg/Aurgasm+Summer+Soundtrack</url> - <title><![CDATA[Aurgasm Summer Soundtrack]]></title> - <artist><![CDATA[Hanne Hukkelberg]]></artist> - <year>0</year> - <popularity>3</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ha%CC%8Avard+Bakke/Emil+fra+L%C3%B8nneberget"> - <url>http://music.yahoo.com/Ha%CC%8Avard+Bakke/Emil+fra+L%C3%B8nneberget</url> - <title><![CDATA[Emil fra Lønneberget]]></title> - <artist><![CDATA[Håvard Bakke]]></artist> - <year>0</year> - <popularity>249</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Hellbillies/R%C3%B8ta_+Hellbillies%27+Beste"> - <url>http://music.yahoo.com/Hellbillies/R%C3%B8ta_+Hellbillies%27+Beste</url> - <title><![CDATA[Røta_ Hellbillies' Beste]]></title> - <artist><![CDATA[Hellbillies]]></artist> - <year>2006</year> - <popularity>14</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/High+Plains+Drifters/NRK+P3+Ur%C3%B8rt"> - <url>http://music.yahoo.com/High+Plains+Drifters/NRK+P3+Ur%C3%B8rt</url> - <title><![CDATA[NRK P3 Urørt]]></title> - <artist><![CDATA[High Plains Drifters]]></artist> - <year>0</year> - <popularity>108</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Holocaust/Unknown+Album"> - <url>http://music.yahoo.com/Holocaust/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Holocaust]]></artist> - <year>0</year> - <popularity>338</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Hopalong+Knut/Feilkalibrert+Tidsmaskin"> - <url>http://music.yahoo.com/Hopalong+Knut/Feilkalibrert+Tidsmaskin</url> - <title><![CDATA[Feilkalibrert Tidsmaskin]]></title> - <artist><![CDATA[Hopalong Knut]]></artist> - <year>2005</year> - <popularity>342</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/I.E/Between+Two+Worlds"> - <url>http://music.yahoo.com/I.E/Between+Two+Worlds</url> - <title><![CDATA[Between Two Worlds]]></title> - <artist><![CDATA[I.E]]></artist> - <year>2006</year> - <popularity>308</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Immortal/Sons+Of+Northern+Darkness"> - <url>http://music.yahoo.com/Immortal/Sons+Of+Northern+Darkness</url> - <title><![CDATA[Sons Of Northern Darkness]]></title> - <artist><![CDATA[Immortal]]></artist> - <year>2002</year> - <popularity>408</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/2_A+Augusti"> - <url>http://music.yahoo.com/Imperiet/2_A+Augusti</url> - <title><![CDATA[2_A Augusti]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>1985</year> - <popularity>384</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/Bla%CC%8A+Himlen+Blues"> - <url>http://music.yahoo.com/Imperiet/Bla%CC%8A+Himlen+Blues</url> - <title><![CDATA[Blå Himlen Blues]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>0</year> - <popularity>307</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/Imperiet"> - <url>http://music.yahoo.com/Imperiet/Imperiet</url> - <title><![CDATA[Imperiet]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>0</year> - <popularity>448</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/Mini-LP+%2B+Rasera"> - <url>http://music.yahoo.com/Imperiet/Mini-LP+%2B+Rasera</url> - <title><![CDATA[Mini-LP + Rasera]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>1992</year> - <popularity>362</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/Synd"> - <url>http://music.yahoo.com/Imperiet/Synd</url> - <title><![CDATA[Synd]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>1986</year> - <popularity>425</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Imperiet/Tiggarens+tal"> - <url>http://music.yahoo.com/Imperiet/Tiggarens+tal</url> - <title><![CDATA[Tiggarens tal]]></title> - <artist><![CDATA[Imperiet]]></artist> - <year>1988</year> - <popularity>346</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Inger+Gundersen/Emma+Og+Thomas+Kommer+Igjen"> - <url>http://music.yahoo.com/Inger+Gundersen/Emma+Og+Thomas+Kommer+Igjen</url> - <title><![CDATA[Emma Og Thomas Kommer Igjen]]></title> - <artist><![CDATA[Inger Gundersen]]></artist> - <year>0</year> - <popularity>151</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Inger+Gundersen/Lilles%C3%B8ster"> - <url>http://music.yahoo.com/Inger+Gundersen/Lilles%C3%B8ster</url> - <title><![CDATA[Lillesøster]]></title> - <artist><![CDATA[Inger Gundersen]]></artist> - <year>2006</year> - <popularity>95</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Inger+Gundersen/Ludde"> - <url>http://music.yahoo.com/Inger+Gundersen/Ludde</url> - <title><![CDATA[Ludde]]></title> - <artist><![CDATA[Inger Gundersen]]></artist> - <year>2002</year> - <popularity>431</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Instinkt/Instinkt"> - <url>http://music.yahoo.com/Instinkt/Instinkt</url> - <title><![CDATA[Instinkt]]></title> - <artist><![CDATA[Instinkt]]></artist> - <year>0</year> - <popularity>418</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Dance+Of+Death"> - <url>http://music.yahoo.com/Iron+Maiden/Dance+Of+Death</url> - <title><![CDATA[Dance Of Death]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>2003</year> - <popularity>442</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Roskilde+2000+%28Extended+Version%29"> - <url>http://music.yahoo.com/Iron+Maiden/Roskilde+2000+%28Extended+Version%29</url> - <title><![CDATA[Roskilde 2000 (Extended Version)]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>2000</year> - <popularity>0</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Unknown+Album"> - <url>http://music.yahoo.com/Iron+Maiden/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>0</year> - <popularity>414</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Israelvis/Eurosis"> - <url>http://music.yahoo.com/Israelvis/Eurosis</url> - <title><![CDATA[Eurosis]]></title> - <artist><![CDATA[Israelvis]]></artist> - <year>0</year> - <popularity>469</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jerry+Cantrell/Degradation+Trip"> - <url>http://music.yahoo.com/Jerry+Cantrell/Degradation+Trip</url> - <title><![CDATA[Degradation Trip]]></title> - <artist><![CDATA[Jerry Cantrell]]></artist> - <year>2002</year> - <popularity>456</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jimi+Hendrix/Band+Of+Gypsys"> - <url>http://music.yahoo.com/Jimi+Hendrix/Band+Of+Gypsys</url> - <title><![CDATA[Band Of Gypsys]]></title> - <artist><![CDATA[Jimi Hendrix]]></artist> - <year>1997</year> - <popularity>455</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/John+Doe/Explosift%21"> - <url>http://music.yahoo.com/John+Doe/Explosift%21</url> - <title><![CDATA[Explosift!]]></title> - <artist><![CDATA[John Doe]]></artist> - <year>0</year> - <popularity>254</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/John+Doe/I+Kveld"> - <url>http://music.yahoo.com/John+Doe/I+Kveld</url> - <title><![CDATA[I Kveld]]></title> - <artist><![CDATA[John Doe]]></artist> - <year>0</year> - <popularity>324</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/John+Doe/Solen"> - <url>http://music.yahoo.com/John+Doe/Solen</url> - <title><![CDATA[Solen]]></title> - <artist><![CDATA[John Doe]]></artist> - <year>0</year> - <popularity>470</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Johndoe/D%C3%B8dvinkel"> - <url>http://music.yahoo.com/Johndoe/D%C3%B8dvinkel</url> - <title><![CDATA[Dødvinkel]]></title> - <artist><![CDATA[Johndoe]]></artist> - <year>0</year> - <popularity>253</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Johnny+Cash/American+III_+Solitary+Man"> - <url>http://music.yahoo.com/Johnny+Cash/American+III_+Solitary+Man</url> - <title><![CDATA[American III_ Solitary Man]]></title> - <artist><![CDATA[Johnny Cash]]></artist> - <year>2000</year> - <popularity>204</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Johnny+Cash/American+IV_+The+Man+Comes+Around"> - <url>http://music.yahoo.com/Johnny+Cash/American+IV_+The+Man+Comes+Around</url> - <title><![CDATA[American IV_ The Man Comes Around]]></title> - <artist><![CDATA[Johnny Cash]]></artist> - <year>2002</year> - <popularity>417</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jokke+%26+Valentinerne/Spenn%21"> - <url>http://music.yahoo.com/Jokke+%26+Valentinerne/Spenn%21</url> - <title><![CDATA[Spenn!]]></title> - <artist><![CDATA[Jokke & Valentinerne]]></artist> - <year>1995</year> - <popularity>156</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jose+Gonzalez/Veneer"> - <url>http://music.yahoo.com/Jose+Gonzalez/Veneer</url> - <title><![CDATA[Veneer]]></title> - <artist><![CDATA[Jose Gonzalez]]></artist> - <year>0</year> - <popularity>58</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Angel+Of+Retribution"> - <url>http://music.yahoo.com/Judas+Priest/Angel+Of+Retribution</url> - <title><![CDATA[Angel Of Retribution]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>2005</year> - <popularity>342</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Defenders+Of+The+Faith+%5BBonus+Tracks%5D"> - <url>http://music.yahoo.com/Judas+Priest/Defenders+Of+The+Faith+%5BBonus+Tracks%5D</url> - <title><![CDATA[Defenders Of The Faith [Bonus Tracks]]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>0</year> - <popularity>239</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Nostradamus"> - <url>http://music.yahoo.com/Judas+Priest/Nostradamus</url> - <title><![CDATA[Nostradamus]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>0</year> - <popularity>250</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Painkiller+%5BRemaster+%2B+Bonus+Tracks%5D"> - <url>http://music.yahoo.com/Judas+Priest/Painkiller+%5BRemaster+%2B+Bonus+Tracks%5D</url> - <title><![CDATA[Painkiller [Remaster + Bonus Tracks]]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>0</year> - <popularity>313</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Priest...Live%21"> - <url>http://music.yahoo.com/Judas+Priest/Priest...Live%21</url> - <title><![CDATA[Priest...Live!]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1987</year> - <popularity>233</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Justin+Timberlake/Justified"> - <url>http://music.yahoo.com/Justin+Timberlake/Justified</url> - <title><![CDATA[Justified]]></title> - <artist><![CDATA[Justin Timberlake]]></artist> - <year>0</year> - <popularity>197</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kai+Remlov/Jentungen+og+s%C3%B8tgr%C3%B8tdagen"> - <url>http://music.yahoo.com/Kai+Remlov/Jentungen+og+s%C3%B8tgr%C3%B8tdagen</url> - <title><![CDATA[Jentungen og søtgrøtdagen]]></title> - <artist><![CDATA[Kai Remlov]]></artist> - <year>0</year> - <popularity>264</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kaptein+Sabeltann/Kaptein+Sabeltann"> - <url>http://music.yahoo.com/Kaptein+Sabeltann/Kaptein+Sabeltann</url> - <title><![CDATA[Kaptein Sabeltann]]></title> - <artist><![CDATA[Kaptein Sabeltann]]></artist> - <year>0</year> - <popularity>27</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Killing+Joke/Unknown+Album"> - <url>http://music.yahoo.com/Killing+Joke/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Killing Joke]]></artist> - <year>0</year> - <popularity>317</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Alive+II"> - <url>http://music.yahoo.com/Kiss/Alive+II</url> - <title><![CDATA[Alive II]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>1977</year> - <popularity>303</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Alive+III+%5BUK%5D+%5BLive%5D"> - <url>http://music.yahoo.com/Kiss/Alive+III+%5BUK%5D+%5BLive%5D</url> - <title><![CDATA[Alive III [UK] [Live]]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>0</year> - <popularity>431</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Alive%21"> - <url>http://music.yahoo.com/Kiss/Alive%21</url> - <title><![CDATA[Alive!]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>1975</year> - <popularity>334</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Five+1992-1999"> - <url>http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Five+1992-1999</url> - <title><![CDATA[Box Set, Disc Five 1992-1999]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>0</year> - <popularity>460</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Four+1983-1989"> - <url>http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Four+1983-1989</url> - <title><![CDATA[Box Set, Disc Four 1983-1989]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>0</year> - <popularity>77</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Three+1976-1982"> - <url>http://music.yahoo.com/Kiss/Box+Set%2C+Disc+Three+1976-1982</url> - <title><![CDATA[Box Set, Disc Three 1976-1982]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>0</year> - <popularity>450</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kiss/Creatures+Of+The+Night"> - <url>http://music.yahoo.com/Kiss/Creatures+Of+The+Night</url> - <title><![CDATA[Creatures Of The Night]]></title> - <artist><![CDATA[Kiss]]></artist> - <year>1982</year> - <popularity>278</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kyuss/And+The+Circus+Leaves+Town"> - <url>http://music.yahoo.com/Kyuss/And+The+Circus+Leaves+Town</url> - <title><![CDATA[And The Circus Leaves Town]]></title> - <artist><![CDATA[Kyuss]]></artist> - <year>1995</year> - <popularity>119</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lumsk/A%CC%8Asmund+Fr%C3%A6gdegjevar"> - <url>http://music.yahoo.com/Lumsk/A%CC%8Asmund+Fr%C3%A6gdegjevar</url> - <title><![CDATA[Åsmund Frægdegjevar]]></title> - <artist><![CDATA[Lumsk]]></artist> - <year>2003</year> - <popularity>69</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lynyrd+Skynyrd/Unknown+Album"> - <url>http://music.yahoo.com/Lynyrd+Skynyrd/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Lynyrd Skynyrd]]></artist> - <year>0</year> - <popularity>128</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mahavishnu+Orchestra/Inner+Mounting+Flame"> - <url>http://music.yahoo.com/Mahavishnu+Orchestra/Inner+Mounting+Flame</url> - <title><![CDATA[Inner Mounting Flame]]></title> - <artist><![CDATA[Mahavishnu Orchestra]]></artist> - <year>0</year> - <popularity>438</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mahavishnu+Orchestra+With+John+McLaughlin/Birds+of+Fire"> - <url>http://music.yahoo.com/Mahavishnu+Orchestra+With+John+McLaughlin/Birds+of+Fire</url> - <title><![CDATA[Birds of Fire]]></title> - <artist><![CDATA[Mahavishnu Orchestra With John McLaughlin]]></artist> - <year>0</year> - <popularity>98</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Malajube/Trompe-L%27Oeil"> - <url>http://music.yahoo.com/Malajube/Trompe-L%27Oeil</url> - <title><![CDATA[Trompe-L'Oeil]]></title> - <artist><![CDATA[Malajube]]></artist> - <year>0</year> - <popularity>279</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Massive+Attack/Mezzanine"> - <url>http://music.yahoo.com/Massive+Attack/Mezzanine</url> - <title><![CDATA[Mezzanine]]></title> - <artist><![CDATA[Massive Attack]]></artist> - <year>0</year> - <popularity>202</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mates+Of+State/Bring+It+Back"> - <url>http://music.yahoo.com/Mates+Of+State/Bring+It+Back</url> - <title><![CDATA[Bring It Back]]></title> - <artist><![CDATA[Mates Of State]]></artist> - <year>0</year> - <popularity>4</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Megadeath/Unknown+Album"> - <url>http://music.yahoo.com/Megadeath/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Megadeath]]></artist> - <year>0</year> - <popularity>279</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mercyful+Fate/Melissa"> - <url>http://music.yahoo.com/Mercyful+Fate/Melissa</url> - <title><![CDATA[Melissa]]></title> - <artist><![CDATA[Mercyful Fate]]></artist> - <year>0</year> - <popularity>260</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mercyful+Fate/Ultimate+Collection"> - <url>http://music.yahoo.com/Mercyful+Fate/Ultimate+Collection</url> - <title><![CDATA[Ultimate Collection]]></title> - <artist><![CDATA[Mercyful Fate]]></artist> - <year>0</year> - <popularity>221</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Death+Magnetic"> - <url>http://music.yahoo.com/Metallica/Death+Magnetic</url> - <title><![CDATA[Death Magnetic]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2008</year> - <popularity>92</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Load"> - <url>http://music.yahoo.com/Metallica/Load</url> - <title><![CDATA[Load]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1996</year> - <popularity>354</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Misc+bootlegs"> - <url>http://music.yahoo.com/Metallica/Misc+bootlegs</url> - <title><![CDATA[Misc bootlegs]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>0</year> - <popularity>152</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Oslo+Overload"> - <url>http://music.yahoo.com/Metallica/Oslo+Overload</url> - <title><![CDATA[Oslo Overload]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>0</year> - <popularity>250</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Oslo+Overload+9_23_96"> - <url>http://music.yahoo.com/Metallica/Oslo+Overload+9_23_96</url> - <title><![CDATA[Oslo Overload 9_23_96]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>0</year> - <popularity>246</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/St.+Anger"> - <url>http://music.yahoo.com/Metallica/St.+Anger</url> - <title><![CDATA[St. Anger]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>367</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Minken+Fosheim/Pippi+Langstr%C3%B8mpe+1"> - <url>http://music.yahoo.com/Minken+Fosheim/Pippi+Langstr%C3%B8mpe+1</url> - <title><![CDATA[Pippi Langstrømpe 1]]></title> - <artist><![CDATA[Minken Fosheim]]></artist> - <year>0</year> - <popularity>51</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Minor+Threat/Unknown"> - <url>http://music.yahoo.com/Minor+Threat/Unknown</url> - <title><![CDATA[Unknown]]></title> - <artist><![CDATA[Minor Threat]]></artist> - <year>0</year> - <popularity>136</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Misfits/Misc"> - <url>http://music.yahoo.com/Misfits/Misc</url> - <title><![CDATA[Misc]]></title> - <artist><![CDATA[Misfits]]></artist> - <year>0</year> - <popularity>426</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motley+Crue/Unknown+Album"> - <url>http://music.yahoo.com/Motley+Crue/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Motley Crue]]></artist> - <year>0</year> - <popularity>28</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorhead/Overkill"> - <url>http://music.yahoo.com/Motorhead/Overkill</url> - <title><![CDATA[Overkill]]></title> - <artist><![CDATA[Motorhead]]></artist> - <year>0</year> - <popularity>450</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorhead/Unknown+Album"> - <url>http://music.yahoo.com/Motorhead/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Motorhead]]></artist> - <year>0</year> - <popularity>360</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Angels+And+Daemons+At+Play"> - <url>http://music.yahoo.com/Motorpsycho/Angels+And+Daemons+At+Play</url> - <title><![CDATA[Angels And Daemons At Play]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>1996</year> - <popularity>205</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Black+Hole+_+Blank+Canvas"> - <url>http://music.yahoo.com/Motorpsycho/Black+Hole+_+Blank+Canvas</url> - <title><![CDATA[Black Hole _ Blank Canvas]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>0</year> - <popularity>101</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Blissard"> - <url>http://music.yahoo.com/Motorpsycho/Blissard</url> - <title><![CDATA[Blissard]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>1996</year> - <popularity>291</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Demon+box"> - <url>http://music.yahoo.com/Motorpsycho/Demon+box</url> - <title><![CDATA[Demon box]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>1993</year> - <popularity>281</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Little+Lucid+Moments"> - <url>http://music.yahoo.com/Motorpsycho/Little+Lucid+Moments</url> - <title><![CDATA[Little Lucid Moments]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>2008</year> - <popularity>326</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Mountain"> - <url>http://music.yahoo.com/Motorpsycho/Mountain</url> - <title><![CDATA[Mountain]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>1993</year> - <popularity>13</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Timothy%27s+Monster"> - <url>http://music.yahoo.com/Motorpsycho/Timothy%27s+Monster</url> - <title><![CDATA[Timothy's Monster]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>1994</year> - <popularity>340</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho+%2B+Deathprod/Toadwork+vol+1_+The+Heavily+Heavenly+Noise+Co-operation"> - <url>http://music.yahoo.com/Motorpsycho+%2B+Deathprod/Toadwork+vol+1_+The+Heavily+Heavenly+Noise+Co-operation</url> - <title><![CDATA[Toadwork vol 1_ The Heavily Heavenly Noise Co-operation]]></title> - <artist><![CDATA[Motorpsycho + Deathprod]]></artist> - <year>0</year> - <popularity>325</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho+tribute/Self+Indulgent+Mono-Minds"> - <url>http://music.yahoo.com/Motorpsycho+tribute/Self+Indulgent+Mono-Minds</url> - <title><![CDATA[Self Indulgent Mono-Minds]]></title> - <artist><![CDATA[Motorpsycho tribute]]></artist> - <year>0</year> - <popularity>375</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Moto%CC%88rhead/Overkill"> - <url>http://music.yahoo.com/Moto%CC%88rhead/Overkill</url> - <title><![CDATA[Overkill]]></title> - <artist><![CDATA[Motörhead]]></artist> - <year>0</year> - <popularity>460</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Moto%CC%88rhead/Stone+Dead+Forever"> - <url>http://music.yahoo.com/Moto%CC%88rhead/Stone+Dead+Forever</url> - <title><![CDATA[Stone Dead Forever]]></title> - <artist><![CDATA[Motörhead]]></artist> - <year>0</year> - <popularity>60</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muse/Origin+Of+Symmetry"> - <url>http://music.yahoo.com/Muse/Origin+Of+Symmetry</url> - <title><![CDATA[Origin Of Symmetry]]></title> - <artist><![CDATA[Muse]]></artist> - <year>2001</year> - <popularity>440</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/My+Chemical+Romance/The+Black+Parade"> - <url>http://music.yahoo.com/My+Chemical+Romance/The+Black+Parade</url> - <title><![CDATA[The Black Parade]]></title> - <artist><![CDATA[My Chemical Romance]]></artist> - <year>0</year> - <popularity>410</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nationalteatern/Barn+av+va%CC%8Ar+tid"> - <url>http://music.yahoo.com/Nationalteatern/Barn+av+va%CC%8Ar+tid</url> - <title><![CDATA[Barn av vår tid]]></title> - <artist><![CDATA[Nationalteatern]]></artist> - <year>0</year> - <popularity>241</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nationalteatern/Ro%CC%88varkungens+o%CC%88"> - <url>http://music.yahoo.com/Nationalteatern/Ro%CC%88varkungens+o%CC%88</url> - <title><![CDATA[Rövarkungens ö]]></title> - <artist><![CDATA[Nationalteatern]]></artist> - <year>0</year> - <popularity>146</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Na%CC%88sblod/Solens+Barn"> - <url>http://music.yahoo.com/Na%CC%88sblod/Solens+Barn</url> - <title><![CDATA[Solens Barn]]></title> - <artist><![CDATA[Näsblod]]></artist> - <year>2003</year> - <popularity>154</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Necrophagist/Epitaph"> - <url>http://music.yahoo.com/Necrophagist/Epitaph</url> - <title><![CDATA[Epitaph]]></title> - <artist><![CDATA[Necrophagist]]></artist> - <year>2004</year> - <popularity>343</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave/Live+Seeds"> - <url>http://music.yahoo.com/Nick+Cave/Live+Seeds</url> - <title><![CDATA[Live Seeds]]></title> - <artist><![CDATA[Nick Cave]]></artist> - <year>1993</year> - <popularity>224</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Abattoir+Blues"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Abattoir+Blues</url> - <title><![CDATA[Abattoir Blues]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>2004</year> - <popularity>299</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Let+Love+In"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Let+Love+In</url> - <title><![CDATA[Let Love In]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1994</year> - <popularity>264</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Murder+Ballads"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Murder+Ballads</url> - <title><![CDATA[Murder Ballads]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1996</year> - <popularity>435</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/No+More+Shall+We+Part"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/No+More+Shall+We+Part</url> - <title><![CDATA[No More Shall We Part]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>2001</year> - <popularity>337</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Good+Son"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Good+Son</url> - <title><![CDATA[The Good Son]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1990</year> - <popularity>154</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Lyre+Of+Orpheus"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Lyre+Of+Orpheus</url> - <title><![CDATA[The Lyre Of Orpheus]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>2004</year> - <popularity>429</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nile/Annihilation+Of+The+Wicked"> - <url>http://music.yahoo.com/Nile/Annihilation+Of+The+Wicked</url> - <title><![CDATA[Annihilation Of The Wicked]]></title> - <artist><![CDATA[Nile]]></artist> - <year>2005</year> - <popularity>410</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nile/In+Their+Darkened+Shrines"> - <url>http://music.yahoo.com/Nile/In+Their+Darkened+Shrines</url> - <title><![CDATA[In Their Darkened Shrines]]></title> - <artist><![CDATA[Nile]]></artist> - <year>2002</year> - <popularity>300</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nile/Ithyphallic"> - <url>http://music.yahoo.com/Nile/Ithyphallic</url> - <title><![CDATA[Ithyphallic]]></title> - <artist><![CDATA[Nile]]></artist> - <year>2007</year> - <popularity>263</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nine+Inch+Nails/Year+Zero"> - <url>http://music.yahoo.com/Nine+Inch+Nails/Year+Zero</url> - <title><![CDATA[Year Zero]]></title> - <artist><![CDATA[Nine Inch Nails]]></artist> - <year>0</year> - <popularity>317</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/No+Doubt/No+Doubt_+The+Singles+%281992-2003%29"> - <url>http://music.yahoo.com/No+Doubt/No+Doubt_+The+Singles+%281992-2003%29</url> - <title><![CDATA[No Doubt_ The Singles (1992-2003)]]></title> - <artist><![CDATA[No Doubt]]></artist> - <year>0</year> - <popularity>101</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/No+Fun+At+All/No+Straight+Angles"> - <url>http://music.yahoo.com/No+Fun+At+All/No+Straight+Angles</url> - <title><![CDATA[No Straight Angles]]></title> - <artist><![CDATA[No Fun At All]]></artist> - <year>1995</year> - <popularity>268</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/No+Fun+At+All/State+Of+Flow"> - <url>http://music.yahoo.com/No+Fun+At+All/State+Of+Flow</url> - <title><![CDATA[State Of Flow]]></title> - <artist><![CDATA[No Fun At All]]></artist> - <year>0</year> - <popularity>344</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/No+Fun+At+All/The+Big+Knockover"> - <url>http://music.yahoo.com/No+Fun+At+All/The+Big+Knockover</url> - <title><![CDATA[The Big Knockover]]></title> - <artist><![CDATA[No Fun At All]]></artist> - <year>1997</year> - <popularity>121</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/NOFX/Punk+In+Drublic"> - <url>http://music.yahoo.com/NOFX/Punk+In+Drublic</url> - <title><![CDATA[Punk In Drublic]]></title> - <artist><![CDATA[NOFX]]></artist> - <year>0</year> - <popularity>83</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/NOFX/Unknown+Album"> - <url>http://music.yahoo.com/NOFX/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[NOFX]]></artist> - <year>0</year> - <popularity>250</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/NRK+P3/NRK+P3+%E2%80%93+Radioresepsjonen"> - <url>http://music.yahoo.com/NRK+P3/NRK+P3+%E2%80%93+Radioresepsjonen</url> - <title><![CDATA[NRK P3 – Radioresepsjonen]]></title> - <artist><![CDATA[NRK P3]]></artist> - <year>0</year> - <popularity>323</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Null%24katte%24nylterne/Skrot"> - <url>http://music.yahoo.com/Null%24katte%24nylterne/Skrot</url> - <title><![CDATA[Skrot]]></title> - <artist><![CDATA[Null$katte$nylterne]]></artist> - <year>0</year> - <popularity>414</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nullskattesnylterne/Fortar+Hardar+H%C3%B8gar"> - <url>http://music.yahoo.com/Nullskattesnylterne/Fortar+Hardar+H%C3%B8gar</url> - <title><![CDATA[Fortar Hardar Høgar]]></title> - <artist><![CDATA[Nullskattesnylterne]]></artist> - <year>2007</year> - <popularity>248</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Numskull/Freezing"> - <url>http://music.yahoo.com/Numskull/Freezing</url> - <title><![CDATA[Freezing]]></title> - <artist><![CDATA[Numskull]]></artist> - <year>0</year> - <popularity>25</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Numskull/S_T"> - <url>http://music.yahoo.com/Numskull/S_T</url> - <title><![CDATA[S_T]]></title> - <artist><![CDATA[Numskull]]></artist> - <year>0</year> - <popularity>484</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Opeth/Watershed"> - <url>http://music.yahoo.com/Opeth/Watershed</url> - <title><![CDATA[Watershed]]></title> - <artist><![CDATA[Opeth]]></artist> - <year>0</year> - <popularity>285</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Peter+Bjorn+And+John/Writer%27s+Block"> - <url>http://music.yahoo.com/Peter+Bjorn+And+John/Writer%27s+Block</url> - <title><![CDATA[Writer's Block]]></title> - <artist><![CDATA[Peter Bjorn And John]]></artist> - <year>0</year> - <popularity>243</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/Delicate+Sound+Of+Thunder"> - <url>http://music.yahoo.com/Pink+Floyd/Delicate+Sound+Of+Thunder</url> - <title><![CDATA[Delicate Sound Of Thunder]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>1988</year> - <popularity>212</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/PJ+Harvey/Is+This+Desire_"> - <url>http://music.yahoo.com/PJ+Harvey/Is+This+Desire_</url> - <title><![CDATA[Is This Desire_]]></title> - <artist><![CDATA[PJ Harvey]]></artist> - <year>1998</year> - <popularity>351</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pony+Up%21/Unknown+Album"> - <url>http://music.yahoo.com/Pony+Up%21/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Pony Up!]]></artist> - <year>0</year> - <popularity>259</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/P%C3%A6stmainn+Pot/Demo"> - <url>http://music.yahoo.com/P%C3%A6stmainn+Pot/Demo</url> - <title><![CDATA[Demo]]></title> - <artist><![CDATA[Pæstmainn Pot]]></artist> - <year>0</year> - <popularity>456</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queen/Unknown+Album"> - <url>http://music.yahoo.com/Queen/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Queen]]></artist> - <year>0</year> - <popularity>215</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Radiohead/OK+Computer"> - <url>http://music.yahoo.com/Radiohead/OK+Computer</url> - <title><![CDATA[OK Computer]]></title> - <artist><![CDATA[Radiohead]]></artist> - <year>1997</year> - <popularity>351</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rage+Against+The+Machine/Evil+Empire"> - <url>http://music.yahoo.com/Rage+Against+The+Machine/Evil+Empire</url> - <title><![CDATA[Evil Empire]]></title> - <artist><![CDATA[Rage Against The Machine]]></artist> - <year>1996</year> - <popularity>285</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rage+Against+The+Machine/Rage+Against+The+Machine"> - <url>http://music.yahoo.com/Rage+Against+The+Machine/Rage+Against+The+Machine</url> - <title><![CDATA[Rage Against The Machine]]></title> - <artist><![CDATA[Rage Against The Machine]]></artist> - <year>1992</year> - <popularity>140</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rage+Against+The+Machine/Renegades"> - <url>http://music.yahoo.com/Rage+Against+The+Machine/Renegades</url> - <title><![CDATA[Renegades]]></title> - <artist><![CDATA[Rage Against The Machine]]></artist> - <year>2000</year> - <popularity>54</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rage+Against+The+Machine/The+Battle+Of+Los+Angeles"> - <url>http://music.yahoo.com/Rage+Against+The+Machine/The+Battle+Of+Los+Angeles</url> - <title><![CDATA[The Battle Of Los Angeles]]></title> - <artist><![CDATA[Rage Against The Machine]]></artist> - <year>1999</year> - <popularity>415</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rama+Lama+Da%CC%88s+Po%CC%88nkabeteilu%CC%88ng/Langpils"> - <url>http://music.yahoo.com/Rama+Lama+Da%CC%88s+Po%CC%88nkabeteilu%CC%88ng/Langpils</url> - <title><![CDATA[Langpils]]></title> - <artist><![CDATA[Rama Lama Däs Pönkabeteilüng]]></artist> - <year>0</year> - <popularity>61</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ravi+%26+DJ+L%C3%B8v/Den+nye+arb%C3%A6idsdagn"> - <url>http://music.yahoo.com/Ravi+%26+DJ+L%C3%B8v/Den+nye+arb%C3%A6idsdagn</url> - <title><![CDATA[Den nye arbæidsdagn]]></title> - <artist><![CDATA[Ravi & DJ Løv]]></artist> - <year>0</year> - <popularity>334</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Red+Hot+Chili+Peppers/Californication"> - <url>http://music.yahoo.com/Red+Hot+Chili+Peppers/Californication</url> - <title><![CDATA[Californication]]></title> - <artist><![CDATA[Red Hot Chili Peppers]]></artist> - <year>1999</year> - <popularity>224</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rockettothesky/To+Sing+You+Apple+Trees"> - <url>http://music.yahoo.com/Rockettothesky/To+Sing+You+Apple+Trees</url> - <title><![CDATA[To Sing You Apple Trees]]></title> - <artist><![CDATA[Rockettothesky]]></artist> - <year>0</year> - <popularity>260</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sambassadeur/Coastal+Affairs"> - <url>http://music.yahoo.com/Sambassadeur/Coastal+Affairs</url> - <title><![CDATA[Coastal Affairs]]></title> - <artist><![CDATA[Sambassadeur]]></artist> - <year>0</year> - <popularity>96</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Savage/Unknown+Album"> - <url>http://music.yahoo.com/Savage/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Savage]]></artist> - <year>0</year> - <popularity>312</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Schto%CC%88ggminator/S_T"> - <url>http://music.yahoo.com/Schto%CC%88ggminator/S_T</url> - <title><![CDATA[S_T]]></title> - <artist><![CDATA[Schtöggminator]]></artist> - <year>0</year> - <popularity>70</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Scissor+Sisters/Scissor+Sisters"> - <url>http://music.yahoo.com/Scissor+Sisters/Scissor+Sisters</url> - <title><![CDATA[Scissor Sisters]]></title> - <artist><![CDATA[Scissor Sisters]]></artist> - <year>0</year> - <popularity>419</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Seid/Among+The+Monster+Flowers+Again"> - <url>http://music.yahoo.com/Seid/Among+The+Monster+Flowers+Again</url> - <title><![CDATA[Among The Monster Flowers Again]]></title> - <artist><![CDATA[Seid]]></artist> - <year>2002</year> - <popularity>81</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Seid/Creatures+Of+The+Underworld"> - <url>http://music.yahoo.com/Seid/Creatures+Of+The+Underworld</url> - <title><![CDATA[Creatures Of The Underworld]]></title> - <artist><![CDATA[Seid]]></artist> - <year>2006</year> - <popularity>33</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigve+B%C3%B8e/Gubben+og+katten+pa%CC%8A+telttur"> - <url>http://music.yahoo.com/Sigve+B%C3%B8e/Gubben+og+katten+pa%CC%8A+telttur</url> - <title><![CDATA[Gubben og katten på telttur]]></title> - <artist><![CDATA[Sigve Bøe]]></artist> - <year>0</year> - <popularity>260</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigve+B%C3%B8e/Mamma+M%C3%B8+Aker+Kjelke"> - <url>http://music.yahoo.com/Sigve+B%C3%B8e/Mamma+M%C3%B8+Aker+Kjelke</url> - <title><![CDATA[Mamma Mø Aker Kjelke]]></title> - <artist><![CDATA[Sigve Bøe]]></artist> - <year>0</year> - <popularity>326</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigve+B%C3%B8e/Mamma+M%C3%B8+I+Byen"> - <url>http://music.yahoo.com/Sigve+B%C3%B8e/Mamma+M%C3%B8+I+Byen</url> - <title><![CDATA[Mamma Mø I Byen]]></title> - <artist><![CDATA[Sigve Bøe]]></artist> - <year>2005</year> - <popularity>400</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigve+B%C3%B8e/Pannekaker%C3%B8re"> - <url>http://music.yahoo.com/Sigve+B%C3%B8e/Pannekaker%C3%B8re</url> - <title><![CDATA[Pannekakerøre]]></title> - <artist><![CDATA[Sigve Bøe]]></artist> - <year>2002</year> - <popularity>212</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigve+B%C3%B8e/Revejakten"> - <url>http://music.yahoo.com/Sigve+B%C3%B8e/Revejakten</url> - <title><![CDATA[Revejakten]]></title> - <artist><![CDATA[Sigve Bøe]]></artist> - <year>0</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Skitsystem/Gra%CC%8A+Va%CC%88rld+_+Svarta+Tankar"> - <url>http://music.yahoo.com/Skitsystem/Gra%CC%8A+Va%CC%88rld+_+Svarta+Tankar</url> - <title><![CDATA[Grå Värld _ Svarta Tankar]]></title> - <artist><![CDATA[Skitsystem]]></artist> - <year>0</year> - <popularity>31</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Slayer/Christ+Illusion+%5BLimited+Edition%5D"> - <url>http://music.yahoo.com/Slayer/Christ+Illusion+%5BLimited+Edition%5D</url> - <title><![CDATA[Christ Illusion [Limited Edition]]]></title> - <artist><![CDATA[Slayer]]></artist> - <year>0</year> - <popularity>270</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Slayer/Reign+In+Blood"> - <url>http://music.yahoo.com/Slayer/Reign+In+Blood</url> - <title><![CDATA[Reign In Blood]]></title> - <artist><![CDATA[Slayer]]></artist> - <year>1986</year> - <popularity>267</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Slipknot/All+Hope+Is+Gone"> - <url>http://music.yahoo.com/Slipknot/All+Hope+Is+Gone</url> - <title><![CDATA[All Hope Is Gone]]></title> - <artist><![CDATA[Slipknot]]></artist> - <year>2008</year> - <popularity>305</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Slipknot/Vol.+3_+%28The+Subliminal+Verses%29"> - <url>http://music.yahoo.com/Slipknot/Vol.+3_+%28The+Subliminal+Verses%29</url> - <title><![CDATA[Vol. 3_ (The Subliminal Verses)]]></title> - <artist><![CDATA[Slipknot]]></artist> - <year>0</year> - <popularity>254</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/StainnR%C3%A6tt/NRK+P3+Ur%C3%B8rt"> - <url>http://music.yahoo.com/StainnR%C3%A6tt/NRK+P3+Ur%C3%B8rt</url> - <title><![CDATA[NRK P3 Urørt]]></title> - <artist><![CDATA[StainnRætt]]></artist> - <year>0</year> - <popularity>387</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Status+Quo/In+the+Army+Now"> - <url>http://music.yahoo.com/Status+Quo/In+the+Army+Now</url> - <title><![CDATA[In the Army Now]]></title> - <artist><![CDATA[Status Quo]]></artist> - <year>0</year> - <popularity>310</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Stone+Deaf+Forever+%28BOX+SET%29+%28/Stone+Deaf+Forever+%28BOX+SET%29+%28Disc+3%29"> - <url>http://music.yahoo.com/Stone+Deaf+Forever+%28BOX+SET%29+%28/Stone+Deaf+Forever+%28BOX+SET%29+%28Disc+3%29</url> - <title><![CDATA[Stone Deaf Forever (BOX SET) (Disc 3)]]></title> - <artist><![CDATA[Stone Deaf Forever (BOX SET) (]]></artist> - <year>0</year> - <popularity>274</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sweet+Savage/Unknown+Album"> - <url>http://music.yahoo.com/Sweet+Savage/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Sweet Savage]]></artist> - <year>0</year> - <popularity>82</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/System+Of+A+Down/Hypnotize"> - <url>http://music.yahoo.com/System+Of+A+Down/Hypnotize</url> - <title><![CDATA[Hypnotize]]></title> - <artist><![CDATA[System Of A Down]]></artist> - <year>2005</year> - <popularity>173</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/System+Of+A+Down/Mezmerize"> - <url>http://music.yahoo.com/System+Of+A+Down/Mezmerize</url> - <title><![CDATA[Mezmerize]]></title> - <artist><![CDATA[System Of A Down]]></artist> - <year>2005</year> - <popularity>107</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/System+Of+A+Down/Steal+This+Album%21"> - <url>http://music.yahoo.com/System+Of+A+Down/Steal+This+Album%21</url> - <title><![CDATA[Steal This Album!]]></title> - <artist><![CDATA[System Of A Down]]></artist> - <year>2002</year> - <popularity>325</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/System+Of+A+Down/Toxicity"> - <url>http://music.yahoo.com/System+Of+A+Down/Toxicity</url> - <title><![CDATA[Toxicity]]></title> - <artist><![CDATA[System Of A Down]]></artist> - <year>2001</year> - <popularity>332</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tadpole/Pole+Position"> - <url>http://music.yahoo.com/Tadpole/Pole+Position</url> - <title><![CDATA[Pole Position]]></title> - <artist><![CDATA[Tadpole]]></artist> - <year>2001</year> - <popularity>270</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tapes+N+Tapes/The+Loon"> - <url>http://music.yahoo.com/Tapes+N+Tapes/The+Loon</url> - <title><![CDATA[The Loon]]></title> - <artist><![CDATA[Tapes N Tapes]]></artist> - <year>0</year> - <popularity>360</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terex+Titan/Demo+1"> - <url>http://music.yahoo.com/Terex+Titan/Demo+1</url> - <title><![CDATA[Demo 1]]></title> - <artist><![CDATA[Terex Titan]]></artist> - <year>0</year> - <popularity>171</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terex+Titan/Demo+2"> - <url>http://music.yahoo.com/Terex+Titan/Demo+2</url> - <title><![CDATA[Demo 2]]></title> - <artist><![CDATA[Terex Titan]]></artist> - <year>0</year> - <popularity>316</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terex+Titan/Demo+3"> - <url>http://music.yahoo.com/Terex+Titan/Demo+3</url> - <title><![CDATA[Demo 3]]></title> - <artist><![CDATA[Terex Titan]]></artist> - <year>0</year> - <popularity>99</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terje+Formoe/Kaptein+Sabeltann+Og+Grusomme+Gabriel%27s+Skatt"> - <url>http://music.yahoo.com/Terje+Formoe/Kaptein+Sabeltann+Og+Grusomme+Gabriel%27s+Skatt</url> - <title><![CDATA[Kaptein Sabeltann Og Grusomme Gabriel's Skatt]]></title> - <artist><![CDATA[Terje Formoe]]></artist> - <year>0</year> - <popularity>113</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terrorizer/Darker+Days+Ahead"> - <url>http://music.yahoo.com/Terrorizer/Darker+Days+Ahead</url> - <title><![CDATA[Darker Days Ahead]]></title> - <artist><![CDATA[Terrorizer]]></artist> - <year>2006</year> - <popularity>98</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Terrorizer/World+Downfall"> - <url>http://music.yahoo.com/Terrorizer/World+Downfall</url> - <title><![CDATA[World Downfall]]></title> - <artist><![CDATA[Terrorizer]]></artist> - <year>1989</year> - <popularity>138</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tha%CC%8Astro%CC%88m/Mannen+Som+Blev+En+Gris"> - <url>http://music.yahoo.com/Tha%CC%8Astro%CC%88m/Mannen+Som+Blev+En+Gris</url> - <title><![CDATA[Mannen Som Blev En Gris]]></title> - <artist><![CDATA[Thåström]]></artist> - <year>2002</year> - <popularity>10</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tha%CC%8Astro%CC%88m/Tha%CC%8Astro%CC%88m"> - <url>http://music.yahoo.com/Tha%CC%8Astro%CC%88m/Tha%CC%8Astro%CC%88m</url> - <title><![CDATA[Thåström]]></title> - <artist><![CDATA[Thåström]]></artist> - <year>1989</year> - <popularity>462</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Arcade+Fire/Funeral"> - <url>http://music.yahoo.com/The+Arcade+Fire/Funeral</url> - <title><![CDATA[Funeral]]></title> - <artist><![CDATA[The Arcade Fire]]></artist> - <year>2004</year> - <popularity>460</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Arcade+Fire/Neon+Bible"> - <url>http://music.yahoo.com/The+Arcade+Fire/Neon+Bible</url> - <title><![CDATA[Neon Bible]]></title> - <artist><![CDATA[The Arcade Fire]]></artist> - <year>2007</year> - <popularity>352</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Darkness/Permission+to+Land+%28Bonus+Track%29"> - <url>http://music.yahoo.com/The+Darkness/Permission+to+Land+%28Bonus+Track%29</url> - <title><![CDATA[Permission to Land (Bonus Track)]]></title> - <artist><![CDATA[The Darkness]]></artist> - <year>0</year> - <popularity>359</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Dubliners/Unknown+Album"> - <url>http://music.yahoo.com/The+Dubliners/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[The Dubliners]]></artist> - <year>0</year> - <popularity>440</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Hobbits/Presidenten+Kommer"> - <url>http://music.yahoo.com/The+Hobbits/Presidenten+Kommer</url> - <title><![CDATA[Presidenten Kommer]]></title> - <artist><![CDATA[The Hobbits]]></artist> - <year>0</year> - <popularity>152</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Hobbits/Rally+EP"> - <url>http://music.yahoo.com/The+Hobbits/Rally+EP</url> - <title><![CDATA[Rally EP]]></title> - <artist><![CDATA[The Hobbits]]></artist> - <year>0</year> - <popularity>29</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Hobbits/Tilt+EP"> - <url>http://music.yahoo.com/The+Hobbits/Tilt+EP</url> - <title><![CDATA[Tilt EP]]></title> - <artist><![CDATA[The Hobbits]]></artist> - <year>0</year> - <popularity>393</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Hold+Steady/Boys+and+Girls+in+America"> - <url>http://music.yahoo.com/The+Hold+Steady/Boys+and+Girls+in+America</url> - <title><![CDATA[Boys and Girls in America]]></title> - <artist><![CDATA[The Hold Steady]]></artist> - <year>0</year> - <popularity>175</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+New+Morty+Show/The+New+Swing+Collection+Jump"> - <url>http://music.yahoo.com/The+New+Morty+Show/The+New+Swing+Collection+Jump</url> - <title><![CDATA[The New Swing Collection Jump]]></title> - <artist><![CDATA[The New Morty Show]]></artist> - <year>0</year> - <popularity>169</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Pipettes/We+Are+The+Pipettes"> - <url>http://music.yahoo.com/The+Pipettes/We+Are+The+Pipettes</url> - <title><![CDATA[We Are The Pipettes]]></title> - <artist><![CDATA[The Pipettes]]></artist> - <year>0</year> - <popularity>403</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Shins/Wincing+The+Night+Away"> - <url>http://music.yahoo.com/The+Shins/Wincing+The+Night+Away</url> - <title><![CDATA[Wincing The Night Away]]></title> - <artist><![CDATA[The Shins]]></artist> - <year>0</year> - <popularity>341</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Soundtrack+of+Our+Lives/Origin+Vol.+1"> - <url>http://music.yahoo.com/The+Soundtrack+of+Our+Lives/Origin+Vol.+1</url> - <title><![CDATA[Origin Vol. 1]]></title> - <artist><![CDATA[The Soundtrack of Our Lives]]></artist> - <year>2004</year> - <popularity>398</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Soundtrack+of+Our+Lives/Welcome+to+the+Infant+Freebase"> - <url>http://music.yahoo.com/The+Soundtrack+of+Our+Lives/Welcome+to+the+Infant+Freebase</url> - <title><![CDATA[Welcome to the Infant Freebase]]></title> - <artist><![CDATA[The Soundtrack of Our Lives]]></artist> - <year>0</year> - <popularity>276</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Thermals/The+Body%2C+The+Blood%2C+The+Machine"> - <url>http://music.yahoo.com/The+Thermals/The+Body%2C+The+Blood%2C+The+Machine</url> - <title><![CDATA[The Body, The Blood, The Machine]]></title> - <artist><![CDATA[The Thermals]]></artist> - <year>0</year> - <popularity>274</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Who/Live+At+Leeds"> - <url>http://music.yahoo.com/The+Who/Live+At+Leeds</url> - <title><![CDATA[Live At Leeds]]></title> - <artist><![CDATA[The Who]]></artist> - <year>1970</year> - <popularity>92</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Thin+Lizzy/Unknown+Album"> - <url>http://music.yahoo.com/Thin+Lizzy/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Thin Lizzy]]></artist> - <year>0</year> - <popularity>123</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Thom+Yorke/The+Eraser"> - <url>http://music.yahoo.com/Thom+Yorke/The+Eraser</url> - <title><![CDATA[The Eraser]]></title> - <artist><![CDATA[Thom Yorke]]></artist> - <year>0</year> - <popularity>417</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Thorbj%C3%B8rn+Egner/Folk+og+r%C3%B8vere+i+Kardemommeby"> - <url>http://music.yahoo.com/Thorbj%C3%B8rn+Egner/Folk+og+r%C3%B8vere+i+Kardemommeby</url> - <title><![CDATA[Folk og røvere i Kardemommeby]]></title> - <artist><![CDATA[Thorbjørn Egner]]></artist> - <year>1955</year> - <popularity>279</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Thorbj%C3%B8rn+Egner/Klatremus+og+de+andre+dyrene+i+Hakkebakkeskogen_"> - <url>http://music.yahoo.com/Thorbj%C3%B8rn+Egner/Klatremus+og+de+andre+dyrene+i+Hakkebakkeskogen_</url> - <title><![CDATA[Klatremus og de andre dyrene i Hakkebakkeskogen_]]></title> - <artist><![CDATA[Thorbjørn Egner]]></artist> - <year>2003</year> - <popularity>488</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Timbuktu/Alla+Vill+Till+Himmelen+Men+Ingen+Vill+Do%CC%88"> - <url>http://music.yahoo.com/Timbuktu/Alla+Vill+Till+Himmelen+Men+Ingen+Vill+Do%CC%88</url> - <title><![CDATA[Alla Vill Till Himmelen Men Ingen Vill Dö]]></title> - <artist><![CDATA[Timbuktu]]></artist> - <year>0</year> - <popularity>6</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Trio+S%C3%B8t/Konsert+Chateau+Neuf"> - <url>http://music.yahoo.com/Trio+S%C3%B8t/Konsert+Chateau+Neuf</url> - <title><![CDATA[Konsert Chateau Neuf]]></title> - <artist><![CDATA[Trio Søt]]></artist> - <year>2005</year> - <popularity>178</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tugboat/Tugboat"> - <url>http://music.yahoo.com/Tugboat/Tugboat</url> - <title><![CDATA[Tugboat]]></title> - <artist><![CDATA[Tugboat]]></artist> - <year>2000</year> - <popularity>231</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Twisted+Sister/Stay+Hungry"> - <url>http://music.yahoo.com/Twisted+Sister/Stay+Hungry</url> - <title><![CDATA[Stay Hungry]]></title> - <artist><![CDATA[Twisted Sister]]></artist> - <year>1984</year> - <popularity>337</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Unknown+Artist/Unknown+Album"> - <url>http://music.yahoo.com/Unknown+Artist/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Unknown Artist]]></artist> - <year>0</year> - <popularity>488</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Vamp/Ma%CC%8Anemannen"> - <url>http://music.yahoo.com/Vamp/Ma%CC%8Anemannen</url> - <title><![CDATA[Månemannen]]></title> - <artist><![CDATA[Vamp]]></artist> - <year>0</year> - <popularity>122</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Vamp/Siste+stikk"> - <url>http://music.yahoo.com/Vamp/Siste+stikk</url> - <title><![CDATA[Siste stikk]]></title> - <artist><![CDATA[Vamp]]></artist> - <year>2005</year> - <popularity>415</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/We/Smugglers"> - <url>http://music.yahoo.com/We/Smugglers</url> - <title><![CDATA[Smugglers]]></title> - <artist><![CDATA[We]]></artist> - <year>2004</year> - <popularity>175</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Willie+Nelson/Across+The+Borderline"> - <url>http://music.yahoo.com/Willie+Nelson/Across+The+Borderline</url> - <title><![CDATA[Across The Borderline]]></title> - <artist><![CDATA[Willie Nelson]]></artist> - <year>1993</year> - <popularity>290</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/%C3%86kte+Vare/NRK+P3+Ur%C3%B8rt"> - <url>http://music.yahoo.com/%C3%86kte+Vare/NRK+P3+Ur%C3%B8rt</url> - <title><![CDATA[NRK P3 Urørt]]></title> - <artist><![CDATA[Ækte Vare]]></artist> - <year>0</year> - <popularity>357</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/120+Days/120+Days"> - <url>http://music.yahoo.com/120+Days/120+Days</url> - <title><![CDATA[120 Days]]></title> - <artist><![CDATA[120 Days]]></artist> - <year>2006</year> - <popularity>72</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/120+Days/Unknown+Album"> - <url>http://music.yahoo.com/120+Days/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[120 Days]]></artist> - <year>0</year> - <popularity>437</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Acid+Grime/Unknown+Album"> - <url>http://music.yahoo.com/Acid+Grime/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Acid Grime]]></artist> - <year>0</year> - <popularity>475</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Animal+Alpha/You+Pay+for+the+Whole+Seat%2C+but+You%27ll+Only+Need+the+Edge"> - <url>http://music.yahoo.com/Animal+Alpha/You+Pay+for+the+Whole+Seat%2C+but+You%27ll+Only+Need+the+Edge</url> - <title><![CDATA[You Pay for the Whole Seat, but You'll Only Need the Edge]]></title> - <artist><![CDATA[Animal Alpha]]></artist> - <year>0</year> - <popularity>436</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Avril+Lavigne/The+Best+Damn+Thing"> - <url>http://music.yahoo.com/Avril+Lavigne/The+Best+Damn+Thing</url> - <title><![CDATA[The Best Damn Thing]]></title> - <artist><![CDATA[Avril Lavigne]]></artist> - <year>2007</year> - <popularity>430</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bertine+Zetlitz/My+Italian+Greyhound"> - <url>http://music.yahoo.com/Bertine+Zetlitz/My+Italian+Greyhound</url> - <title><![CDATA[My Italian Greyhound]]></title> - <artist><![CDATA[Bertine Zetlitz]]></artist> - <year>2006</year> - <popularity>248</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bertine+Zetlitz/Rollerskating"> - <url>http://music.yahoo.com/Bertine+Zetlitz/Rollerskating</url> - <title><![CDATA[Rollerskating]]></title> - <artist><![CDATA[Bertine Zetlitz]]></artist> - <year>2004</year> - <popularity>330</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bertine+Zetlitz/Sweet+injections"> - <url>http://music.yahoo.com/Bertine+Zetlitz/Sweet+injections</url> - <title><![CDATA[Sweet injections]]></title> - <artist><![CDATA[Bertine Zetlitz]]></artist> - <year>2003</year> - <popularity>184</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bjo%CC%88rk/Post"> - <url>http://music.yahoo.com/Bjo%CC%88rk/Post</url> - <title><![CDATA[Post]]></title> - <artist><![CDATA[Björk]]></artist> - <year>1995</year> - <popularity>92</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Debbath/Tung%2C+tung+politisk+rock"> - <url>http://music.yahoo.com/Black+Debbath/Tung%2C+tung+politisk+rock</url> - <title><![CDATA[Tung, tung politisk rock]]></title> - <artist><![CDATA[Black Debbath]]></artist> - <year>1999</year> - <popularity>87</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Sabbath/Greatest+Hits"> - <url>http://music.yahoo.com/Black+Sabbath/Greatest+Hits</url> - <title><![CDATA[Greatest Hits]]></title> - <artist><![CDATA[Black Sabbath]]></artist> - <year>1977</year> - <popularity>318</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Sabbath/Heaven+And+Hell"> - <url>http://music.yahoo.com/Black+Sabbath/Heaven+And+Hell</url> - <title><![CDATA[Heaven And Hell]]></title> - <artist><![CDATA[Black Sabbath]]></artist> - <year>1980</year> - <popularity>317</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Black+Sabbath/The+Best+Of+Black+Sabbath"> - <url>http://music.yahoo.com/Black+Sabbath/The+Best+Of+Black+Sabbath</url> - <title><![CDATA[The Best Of Black Sabbath]]></title> - <artist><![CDATA[Black Sabbath]]></artist> - <year>2000</year> - <popularity>264</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bobby+Martini/Killer+Client"> - <url>http://music.yahoo.com/Bobby+Martini/Killer+Client</url> - <title><![CDATA[Killer Client]]></title> - <artist><![CDATA[Bobby Martini]]></artist> - <year>0</year> - <popularity>248</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Britney+Spears/Greatest+Hits_+My+Prerogative"> - <url>http://music.yahoo.com/Britney+Spears/Greatest+Hits_+My+Prerogative</url> - <title><![CDATA[Greatest Hits_ My Prerogative]]></title> - <artist><![CDATA[Britney Spears]]></artist> - <year>2004</year> - <popularity>121</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/BT/This+Binary+Universe"> - <url>http://music.yahoo.com/BT/This+Binary+Universe</url> - <title><![CDATA[This Binary Universe]]></title> - <artist><![CDATA[BT]]></artist> - <year>2006</year> - <popularity>147</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/City"> - <url>http://music.yahoo.com/Client/City</url> - <title><![CDATA[City]]></title> - <artist><![CDATA[Client]]></artist> - <year>2004</year> - <popularity>421</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Client"> - <url>http://music.yahoo.com/Client/Client</url> - <title><![CDATA[Client]]></title> - <artist><![CDATA[Client]]></artist> - <year>2003</year> - <popularity>52</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Client+Members+Forum"> - <url>http://music.yahoo.com/Client/Client+Members+Forum</url> - <title><![CDATA[Client Members Forum]]></title> - <artist><![CDATA[Client]]></artist> - <year>0</year> - <popularity>121</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Client+Shrine"> - <url>http://music.yahoo.com/Client/Client+Shrine</url> - <title><![CDATA[Client Shrine]]></title> - <artist><![CDATA[Client]]></artist> - <year>0</year> - <popularity>413</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Drive"> - <url>http://music.yahoo.com/Client/Drive</url> - <title><![CDATA[Drive]]></title> - <artist><![CDATA[Client]]></artist> - <year>2006</year> - <popularity>201</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Heartland"> - <url>http://music.yahoo.com/Client/Heartland</url> - <title><![CDATA[Heartland]]></title> - <artist><![CDATA[Client]]></artist> - <year>2007</year> - <popularity>400</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Here+And+Now"> - <url>http://music.yahoo.com/Client/Here+And+Now</url> - <title><![CDATA[Here And Now]]></title> - <artist><![CDATA[Client]]></artist> - <year>2003</year> - <popularity>322</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/In+It+For+The+Money"> - <url>http://music.yahoo.com/Client/In+It+For+The+Money</url> - <title><![CDATA[In It For The Money]]></title> - <artist><![CDATA[Client]]></artist> - <year>2004</year> - <popularity>485</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/In+It+For+The+Money+%28Single%29"> - <url>http://music.yahoo.com/Client/In+It+For+The+Money+%28Single%29</url> - <title><![CDATA[In It For The Money (Single)]]></title> - <artist><![CDATA[Client]]></artist> - <year>2004</year> - <popularity>372</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/It%27s+Not+Over"> - <url>http://music.yahoo.com/Client/It%27s+Not+Over</url> - <title><![CDATA[It's Not Over]]></title> - <artist><![CDATA[Client]]></artist> - <year>2007</year> - <popularity>187</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Lights+Go+Out"> - <url>http://music.yahoo.com/Client/Lights+Go+Out</url> - <title><![CDATA[Lights Go Out]]></title> - <artist><![CDATA[Client]]></artist> - <year>2006</year> - <popularity>231</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Live+In+Porto"> - <url>http://music.yahoo.com/Client/Live+In+Porto</url> - <title><![CDATA[Live In Porto]]></title> - <artist><![CDATA[Client]]></artist> - <year>0</year> - <popularity>53</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Metropolis"> - <url>http://music.yahoo.com/Client/Metropolis</url> - <title><![CDATA[Metropolis]]></title> - <artist><![CDATA[Client]]></artist> - <year>2005</year> - <popularity>51</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Pornography+%28Disc+2%29"> - <url>http://music.yahoo.com/Client/Pornography+%28Disc+2%29</url> - <title><![CDATA[Pornography (Disc 2)]]></title> - <artist><![CDATA[Client]]></artist> - <year>2005</year> - <popularity>484</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Radio+-+EP"> - <url>http://music.yahoo.com/Client/Radio+-+EP</url> - <title><![CDATA[Radio - EP]]></title> - <artist><![CDATA[Client]]></artist> - <year>2004</year> - <popularity>205</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Rock+And+Roll+Machine"> - <url>http://music.yahoo.com/Client/Rock+And+Roll+Machine</url> - <title><![CDATA[Rock And Roll Machine]]></title> - <artist><![CDATA[Client]]></artist> - <year>2003</year> - <popularity>226</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Client/Zerox+Machine"> - <url>http://music.yahoo.com/Client/Zerox+Machine</url> - <title><![CDATA[Zerox Machine]]></title> - <artist><![CDATA[Client]]></artist> - <year>2007</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/A+Clockwork+Orange"> - <url>http://music.yahoo.com/Compilations/A+Clockwork+Orange</url> - <title><![CDATA[A Clockwork Orange]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>1972</year> - <popularity>146</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Ed+Rec+Vol.2+%28Ed+Banger+Records%29"> - <url>http://music.yahoo.com/Compilations/Ed+Rec+Vol.2+%28Ed+Banger+Records%29</url> - <title><![CDATA[Ed Rec Vol.2 (Ed Banger Records)]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>2007</year> - <popularity>481</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/M_I-2"> - <url>http://music.yahoo.com/Compilations/M_I-2</url> - <title><![CDATA[M_I-2]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>2000</year> - <popularity>187</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Covenant/In+Transit"> - <url>http://music.yahoo.com/Covenant/In+Transit</url> - <title><![CDATA[In Transit]]></title> - <artist><![CDATA[Covenant]]></artist> - <year>2007</year> - <popularity>397</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Daft+Punk/Alive+2007"> - <url>http://music.yahoo.com/Daft+Punk/Alive+2007</url> - <title><![CDATA[Alive 2007]]></title> - <artist><![CDATA[Daft Punk]]></artist> - <year>2007</year> - <popularity>101</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Daft+Punk/Discovery"> - <url>http://music.yahoo.com/Daft+Punk/Discovery</url> - <title><![CDATA[Discovery]]></title> - <artist><![CDATA[Daft Punk]]></artist> - <year>2001</year> - <popularity>456</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Daft+Punk/Musique+Vol.+1+1993-2005"> - <url>http://music.yahoo.com/Daft+Punk/Musique+Vol.+1+1993-2005</url> - <title><![CDATA[Musique Vol. 1 1993-2005]]></title> - <artist><![CDATA[Daft Punk]]></artist> - <year>1996</year> - <popularity>413</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Datarock/Datarock+Datarock"> - <url>http://music.yahoo.com/Datarock/Datarock+Datarock</url> - <title><![CDATA[Datarock Datarock]]></title> - <artist><![CDATA[Datarock]]></artist> - <year>2005</year> - <popularity>120</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/De+Lillos/Mere"> - <url>http://music.yahoo.com/De+Lillos/Mere</url> - <title><![CDATA[Mere]]></title> - <artist><![CDATA[De Lillos]]></artist> - <year>1994</year> - <popularity>192</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/deathprod/imaginary+songs+from+tristan+da+cunha"> - <url>http://music.yahoo.com/deathprod/imaginary+songs+from+tristan+da+cunha</url> - <title><![CDATA[imaginary songs from tristan da cunha]]></title> - <artist><![CDATA[deathprod]]></artist> - <year>0</year> - <popularity>201</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/deathprod/morals+and+dogma"> - <url>http://music.yahoo.com/deathprod/morals+and+dogma</url> - <title><![CDATA[morals and dogma]]></title> - <artist><![CDATA[deathprod]]></artist> - <year>0</year> - <popularity>74</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/deathprod/reference+frequencies"> - <url>http://music.yahoo.com/deathprod/reference+frequencies</url> - <title><![CDATA[reference frequencies]]></title> - <artist><![CDATA[deathprod]]></artist> - <year>0</year> - <popularity>393</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/deathprod/treetop+drive"> - <url>http://music.yahoo.com/deathprod/treetop+drive</url> - <title><![CDATA[treetop drive]]></title> - <artist><![CDATA[deathprod]]></artist> - <year>0</year> - <popularity>94</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Behind+The+Wheel+%28Remixes%29+Single"> - <url>http://music.yahoo.com/Depeche+Mode/Behind+The+Wheel+%28Remixes%29+Single</url> - <title><![CDATA[Behind The Wheel (Remixes) Single]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1988</year> - <popularity>341</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Playing+The+Angel"> - <url>http://music.yahoo.com/Depeche+Mode/Playing+The+Angel</url> - <title><![CDATA[Playing The Angel]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>2005</year> - <popularity>49</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Policy+Of+Truth+%28Remixes%29+%28PRO-CD-4027%29"> - <url>http://music.yahoo.com/Depeche+Mode/Policy+Of+Truth+%28Remixes%29+%28PRO-CD-4027%29</url> - <title><![CDATA[Policy Of Truth (Remixes) (PRO-CD-4027)]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1990</year> - <popularity>150</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Remixes+81-04"> - <url>http://music.yahoo.com/Depeche+Mode/Remixes+81-04</url> - <title><![CDATA[Remixes 81-04]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>2004</year> - <popularity>188</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Ultra"> - <url>http://music.yahoo.com/Depeche+Mode/Ultra</url> - <title><![CDATA[Ultra]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1997</year> - <popularity>179</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Violator"> - <url>http://music.yahoo.com/Depeche+Mode/Violator</url> - <title><![CDATA[Violator]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1990</year> - <popularity>371</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/Walking+In+My+Shoes"> - <url>http://music.yahoo.com/Depeche+Mode/Walking+In+My+Shoes</url> - <title><![CDATA[Walking In My Shoes]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1993</year> - <popularity>429</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Depeche+Mode/World+In+My+Eyes+%28Japan%29"> - <url>http://music.yahoo.com/Depeche+Mode/World+In+My+Eyes+%28Japan%29</url> - <title><![CDATA[World In My Eyes (Japan)]]></title> - <artist><![CDATA[Depeche Mode]]></artist> - <year>1990</year> - <popularity>184</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dimmu+Borgir/Death+Cult+Armageddon"> - <url>http://music.yahoo.com/Dimmu+Borgir/Death+Cult+Armageddon</url> - <title><![CDATA[Death Cult Armageddon]]></title> - <artist><![CDATA[Dimmu Borgir]]></artist> - <year>2003</year> - <popularity>96</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dimmu+Borgir/Enthrone+Darkness+Triumphant"> - <url>http://music.yahoo.com/Dimmu+Borgir/Enthrone+Darkness+Triumphant</url> - <title><![CDATA[Enthrone Darkness Triumphant]]></title> - <artist><![CDATA[Dimmu Borgir]]></artist> - <year>1997</year> - <popularity>135</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dimmu+Borgir/Puritanical+Euphoric+Misanthropia"> - <url>http://music.yahoo.com/Dimmu+Borgir/Puritanical+Euphoric+Misanthropia</url> - <title><![CDATA[Puritanical Euphoric Misanthropia]]></title> - <artist><![CDATA[Dimmu Borgir]]></artist> - <year>2001</year> - <popularity>158</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dimmu+Borgir/Spiritual+Black+Dimensions"> - <url>http://music.yahoo.com/Dimmu+Borgir/Spiritual+Black+Dimensions</url> - <title><![CDATA[Spiritual Black Dimensions]]></title> - <artist><![CDATA[Dimmu Borgir]]></artist> - <year>1999</year> - <popularity>305</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dimmu+Borgir/Stormbla%CC%8Ast"> - <url>http://music.yahoo.com/Dimmu+Borgir/Stormbla%CC%8Ast</url> - <title><![CDATA[Stormblåst]]></title> - <artist><![CDATA[Dimmu Borgir]]></artist> - <year>2005</year> - <popularity>398</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dio/Diamonds_+The+Best+Of+Dio"> - <url>http://music.yahoo.com/Dio/Diamonds_+The+Best+Of+Dio</url> - <title><![CDATA[Diamonds_ The Best Of Dio]]></title> - <artist><![CDATA[Dio]]></artist> - <year>1994</year> - <popularity>482</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dire+Straits/Brothers+In+Arms"> - <url>http://music.yahoo.com/Dire+Straits/Brothers+In+Arms</url> - <title><![CDATA[Brothers In Arms]]></title> - <artist><![CDATA[Dire Straits]]></artist> - <year>1985</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dubstar/Make+It+Better"> - <url>http://music.yahoo.com/Dubstar/Make+It+Better</url> - <title><![CDATA[Make It Better]]></title> - <artist><![CDATA[Dubstar]]></artist> - <year>2000</year> - <popularity>151</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eminem/Encore"> - <url>http://music.yahoo.com/Eminem/Encore</url> - <title><![CDATA[Encore]]></title> - <artist><![CDATA[Eminem]]></artist> - <year>2004</year> - <popularity>235</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Erasure/ABBA-esque"> - <url>http://music.yahoo.com/Erasure/ABBA-esque</url> - <title><![CDATA[ABBA-esque]]></title> - <artist><![CDATA[Erasure]]></artist> - <year>1992</year> - <popularity>84</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Erykah+Badu/Baduizm"> - <url>http://music.yahoo.com/Erykah+Badu/Baduizm</url> - <title><![CDATA[Baduizm]]></title> - <artist><![CDATA[Erykah Badu]]></artist> - <year>1997</year> - <popularity>236</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Erykah+Badu/Mama%27s+Gun"> - <url>http://music.yahoo.com/Erykah+Badu/Mama%27s+Gun</url> - <title><![CDATA[Mama's Gun]]></title> - <artist><![CDATA[Erykah Badu]]></artist> - <year>2000</year> - <popularity>235</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eva+Cassidy/Songbird"> - <url>http://music.yahoo.com/Eva+Cassidy/Songbird</url> - <title><![CDATA[Songbird]]></title> - <artist><![CDATA[Eva Cassidy]]></artist> - <year>1998</year> - <popularity>138</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eva+Cassidy/Time+After+Time"> - <url>http://music.yahoo.com/Eva+Cassidy/Time+After+Time</url> - <title><![CDATA[Time After Time]]></title> - <artist><![CDATA[Eva Cassidy]]></artist> - <year>2000</year> - <popularity>76</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Faithless/No+Roots"> - <url>http://music.yahoo.com/Faithless/No+Roots</url> - <title><![CDATA[No Roots]]></title> - <artist><![CDATA[Faithless]]></artist> - <year>2004</year> - <popularity>408</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Fatboy+Slim/You%27ve+Come+a+Long+Way%2C+Baby"> - <url>http://music.yahoo.com/Fatboy+Slim/You%27ve+Come+a+Long+Way%2C+Baby</url> - <title><![CDATA[You've Come a Long Way, Baby]]></title> - <artist><![CDATA[Fatboy Slim]]></artist> - <year>0</year> - <popularity>210</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Flo+Rida/Low+%28feat.+T-Pain%29+-+EP"> - <url>http://music.yahoo.com/Flo+Rida/Low+%28feat.+T-Pain%29+-+EP</url> - <title><![CDATA[Low (feat. T-Pain) - EP]]></title> - <artist><![CDATA[Flo Rida]]></artist> - <year>2007</year> - <popularity>53</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/FM3/Buddha+Machine"> - <url>http://music.yahoo.com/FM3/Buddha+Machine</url> - <title><![CDATA[Buddha Machine]]></title> - <artist><![CDATA[FM3]]></artist> - <year>2006</year> - <popularity>418</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Foo+Fighters/Echoes%2C+Silence%2C+Patience+%26+Grace"> - <url>http://music.yahoo.com/Foo+Fighters/Echoes%2C+Silence%2C+Patience+%26+Grace</url> - <title><![CDATA[Echoes, Silence, Patience & Grace]]></title> - <artist><![CDATA[Foo Fighters]]></artist> - <year>2007</year> - <popularity>346</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Frankie+Goes+To+Hollywood/Rage+Hard_+The+Sonic+Collection"> - <url>http://music.yahoo.com/Frankie+Goes+To+Hollywood/Rage+Hard_+The+Sonic+Collection</url> - <title><![CDATA[Rage Hard_ The Sonic Collection]]></title> - <artist><![CDATA[Frankie Goes To Hollywood]]></artist> - <year>1984</year> - <popularity>456</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Frankie+Goes+To+Hollywood/Welcome+To+The+Pleasuredome"> - <url>http://music.yahoo.com/Frankie+Goes+To+Hollywood/Welcome+To+The+Pleasuredome</url> - <title><![CDATA[Welcome To The Pleasuredome]]></title> - <artist><![CDATA[Frankie Goes To Hollywood]]></artist> - <year>1984</year> - <popularity>149</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Front+242/05_22_09_12+Off"> - <url>http://music.yahoo.com/Front+242/05_22_09_12+Off</url> - <title><![CDATA[05_22_09_12 Off]]></title> - <artist><![CDATA[Front 242]]></artist> - <year>1993</year> - <popularity>462</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Garbage/Version+2.0"> - <url>http://music.yahoo.com/Garbage/Version+2.0</url> - <title><![CDATA[Version 2.0]]></title> - <artist><![CDATA[Garbage]]></artist> - <year>1998</year> - <popularity>193</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gary+Moore/Out+In+The+Fields+-+The+Very+Best+Of+Gary+Moore"> - <url>http://music.yahoo.com/Gary+Moore/Out+In+The+Fields+-+The+Very+Best+Of+Gary+Moore</url> - <title><![CDATA[Out In The Fields - The Very Best Of Gary Moore]]></title> - <artist><![CDATA[Gary Moore]]></artist> - <year>1998</year> - <popularity>42</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ga%CC%8Ate/Iselilja"> - <url>http://music.yahoo.com/Ga%CC%8Ate/Iselilja</url> - <title><![CDATA[Iselilja]]></title> - <artist><![CDATA[Gåte]]></artist> - <year>2004</year> - <popularity>170</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Girl+Talk/Feed+the+Animals"> - <url>http://music.yahoo.com/Girl+Talk/Feed+the+Animals</url> - <title><![CDATA[Feed the Animals]]></title> - <artist><![CDATA[Girl Talk]]></artist> - <year>0</year> - <popularity>249</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gorillaz/Gorillaz"> - <url>http://music.yahoo.com/Gorillaz/Gorillaz</url> - <title><![CDATA[Gorillaz]]></title> - <artist><![CDATA[Gorillaz]]></artist> - <year>2001</year> - <popularity>348</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Guns+N%27+Roses/Appetite+For+Destruction"> - <url>http://music.yahoo.com/Guns+N%27+Roses/Appetite+For+Destruction</url> - <title><![CDATA[Appetite For Destruction]]></title> - <artist><![CDATA[Guns N' Roses]]></artist> - <year>1987</year> - <popularity>382</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/IAMX/Nightlife"> - <url>http://music.yahoo.com/IAMX/Nightlife</url> - <title><![CDATA[Nightlife]]></title> - <artist><![CDATA[IAMX]]></artist> - <year>2007</year> - <popularity>256</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/A+Matter+Of+Life+And+Death"> - <url>http://music.yahoo.com/Iron+Maiden/A+Matter+Of+Life+And+Death</url> - <title><![CDATA[A Matter Of Life And Death]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>2006</year> - <popularity>463</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Fear+Of+The+Dark"> - <url>http://music.yahoo.com/Iron+Maiden/Fear+Of+The+Dark</url> - <title><![CDATA[Fear Of The Dark]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1992</year> - <popularity>477</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Iron+Maiden"> - <url>http://music.yahoo.com/Iron+Maiden/Iron+Maiden</url> - <title><![CDATA[Iron Maiden]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1981</year> - <popularity>171</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Killers"> - <url>http://music.yahoo.com/Iron+Maiden/Killers</url> - <title><![CDATA[Killers]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1981</year> - <popularity>14</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Live+After+Death"> - <url>http://music.yahoo.com/Iron+Maiden/Live+After+Death</url> - <title><![CDATA[Live After Death]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1985</year> - <popularity>253</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/No+Prayer+For+The+Dying"> - <url>http://music.yahoo.com/Iron+Maiden/No+Prayer+For+The+Dying</url> - <title><![CDATA[No Prayer For The Dying]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1990</year> - <popularity>319</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Piece+Of+Mind"> - <url>http://music.yahoo.com/Iron+Maiden/Piece+Of+Mind</url> - <title><![CDATA[Piece Of Mind]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1983</year> - <popularity>25</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Powerslave"> - <url>http://music.yahoo.com/Iron+Maiden/Powerslave</url> - <title><![CDATA[Powerslave]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1984</year> - <popularity>496</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Seventh+Son+Of+A+Seventh+Son"> - <url>http://music.yahoo.com/Iron+Maiden/Seventh+Son+Of+A+Seventh+Son</url> - <title><![CDATA[Seventh Son Of A Seventh Son]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1988</year> - <popularity>181</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Somewhere+Back+In+Time_+The+Best+Of_+1980-1989"> - <url>http://music.yahoo.com/Iron+Maiden/Somewhere+Back+In+Time_+The+Best+Of_+1980-1989</url> - <title><![CDATA[Somewhere Back In Time_ The Best Of_ 1980-1989]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>2008</year> - <popularity>496</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Somewhere+In+Time"> - <url>http://music.yahoo.com/Iron+Maiden/Somewhere+In+Time</url> - <title><![CDATA[Somewhere In Time]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1986</year> - <popularity>473</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/The+Number+Of+The+Beast"> - <url>http://music.yahoo.com/Iron+Maiden/The+Number+Of+The+Beast</url> - <title><![CDATA[The Number Of The Beast]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1982</year> - <popularity>303</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Iron+Maiden/Virtual+XI"> - <url>http://music.yahoo.com/Iron+Maiden/Virtual+XI</url> - <title><![CDATA[Virtual XI]]></title> - <artist><![CDATA[Iron Maiden]]></artist> - <year>1998</year> - <popularity>291</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jane%27s+Addiction/Ritual+De+Lo+Habitual"> - <url>http://music.yahoo.com/Jane%27s+Addiction/Ritual+De+Lo+Habitual</url> - <title><![CDATA[Ritual De Lo Habitual]]></title> - <artist><![CDATA[Jane's Addiction]]></artist> - <year>1990</year> - <popularity>137</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jay+Lake/Unknown+Album"> - <url>http://music.yahoo.com/Jay+Lake/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Jay Lake]]></artist> - <year>0</year> - <popularity>432</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/British+Steel"> - <url>http://music.yahoo.com/Judas+Priest/British+Steel</url> - <title><![CDATA[British Steel]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1980</year> - <popularity>460</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Defenders+Of+The+Faith"> - <url>http://music.yahoo.com/Judas+Priest/Defenders+Of+The+Faith</url> - <title><![CDATA[Defenders Of The Faith]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>2002</year> - <popularity>20</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Hell+Bent+For+Leather"> - <url>http://music.yahoo.com/Judas+Priest/Hell+Bent+For+Leather</url> - <title><![CDATA[Hell Bent For Leather]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1979</year> - <popularity>233</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Living+After+Midnight+-The+Best+Of+Judas+Priest"> - <url>http://music.yahoo.com/Judas+Priest/Living+After+Midnight+-The+Best+Of+Judas+Priest</url> - <title><![CDATA[Living After Midnight -The Best Of Judas Priest]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1997</year> - <popularity>440</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Metal+Works+%2773-%2793"> - <url>http://music.yahoo.com/Judas+Priest/Metal+Works+%2773-%2793</url> - <title><![CDATA[Metal Works '73-'93]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1982</year> - <popularity>151</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Screaming+For+Vengeance"> - <url>http://music.yahoo.com/Judas+Priest/Screaming+For+Vengeance</url> - <title><![CDATA[Screaming For Vengeance]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1982</year> - <popularity>198</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Judas+Priest/Sin+After+Sin"> - <url>http://music.yahoo.com/Judas+Priest/Sin+After+Sin</url> - <title><![CDATA[Sin After Sin]]></title> - <artist><![CDATA[Judas Priest]]></artist> - <year>1977</year> - <popularity>373</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Juno+Reactor/Bible+Of+Dreams"> - <url>http://music.yahoo.com/Juno+Reactor/Bible+Of+Dreams</url> - <title><![CDATA[Bible Of Dreams]]></title> - <artist><![CDATA[Juno Reactor]]></artist> - <year>1997</year> - <popularity>151</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Juno+Reactor/Labyrinth"> - <url>http://music.yahoo.com/Juno+Reactor/Labyrinth</url> - <title><![CDATA[Labyrinth]]></title> - <artist><![CDATA[Juno Reactor]]></artist> - <year>2004</year> - <popularity>299</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Juno+Reactor/Shango"> - <url>http://music.yahoo.com/Juno+Reactor/Shango</url> - <title><![CDATA[Shango]]></title> - <artist><![CDATA[Juno Reactor]]></artist> - <year>2000</year> - <popularity>15</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Katy+Perry/I+Kissed+a+Girl+-+Single"> - <url>http://music.yahoo.com/Katy+Perry/I+Kissed+a+Girl+-+Single</url> - <title><![CDATA[I Kissed a Girl - Single]]></title> - <artist><![CDATA[Katy Perry]]></artist> - <year>0</year> - <popularity>414</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Keane/Hopes+And+Fears"> - <url>http://music.yahoo.com/Keane/Hopes+And+Fears</url> - <title><![CDATA[Hopes And Fears]]></title> - <artist><![CDATA[Keane]]></artist> - <year>2004</year> - <popularity>299</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Keep+Of+Kalessin/Armada"> - <url>http://music.yahoo.com/Keep+Of+Kalessin/Armada</url> - <title><![CDATA[Armada]]></title> - <artist><![CDATA[Keep Of Kalessin]]></artist> - <year>2006</year> - <popularity>190</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/B-Sidor+95-00"> - <url>http://music.yahoo.com/Kent/B-Sidor+95-00</url> - <title><![CDATA[B-Sidor 95-00]]></title> - <artist><![CDATA[Kent]]></artist> - <year>2000</year> - <popularity>171</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/Isola"> - <url>http://music.yahoo.com/Kent/Isola</url> - <title><![CDATA[Isola]]></title> - <artist><![CDATA[Kent]]></artist> - <year>1997</year> - <popularity>221</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/Kent"> - <url>http://music.yahoo.com/Kent/Kent</url> - <title><![CDATA[Kent]]></title> - <artist><![CDATA[Kent]]></artist> - <year>1995</year> - <popularity>467</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/Tillbaka+till+samtiden"> - <url>http://music.yahoo.com/Kent/Tillbaka+till+samtiden</url> - <title><![CDATA[Tillbaka till samtiden]]></title> - <artist><![CDATA[Kent]]></artist> - <year>2007</year> - <popularity>260</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/Vapen+%26+Ammunition"> - <url>http://music.yahoo.com/Kent/Vapen+%26+Ammunition</url> - <title><![CDATA[Vapen & Ammunition]]></title> - <artist><![CDATA[Kent]]></artist> - <year>2002</year> - <popularity>231</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kent/Verkligen"> - <url>http://music.yahoo.com/Kent/Verkligen</url> - <title><![CDATA[Verkligen]]></title> - <artist><![CDATA[Kent]]></artist> - <year>1996</year> - <popularity>192</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kraftwerk/The+Man+Machine"> - <url>http://music.yahoo.com/Kraftwerk/The+Man+Machine</url> - <title><![CDATA[The Man Machine]]></title> - <artist><![CDATA[Kraftwerk]]></artist> - <year>1978</year> - <popularity>422</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kraftwerk/The+Mix"> - <url>http://music.yahoo.com/Kraftwerk/The+Mix</url> - <title><![CDATA[The Mix]]></title> - <artist><![CDATA[Kraftwerk]]></artist> - <year>1991</year> - <popularity>203</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kylie+Minogue/Fever"> - <url>http://music.yahoo.com/Kylie+Minogue/Fever</url> - <title><![CDATA[Fever]]></title> - <artist><![CDATA[Kylie Minogue]]></artist> - <year>2001</year> - <popularity>329</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/604"> - <url>http://music.yahoo.com/Ladytron/604</url> - <title><![CDATA[604]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>2001</year> - <popularity>215</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Extended+Play"> - <url>http://music.yahoo.com/Ladytron/Extended+Play</url> - <title><![CDATA[Extended Play]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>2006</year> - <popularity>85</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Ghosts+%28Blestenation+Mix%29+-+Get+Free+Music+at+RCRD+LBL.com"> - <url>http://music.yahoo.com/Ladytron/Ghosts+%28Blestenation+Mix%29+-+Get+Free+Music+at+RCRD+LBL.com</url> - <title><![CDATA[Ghosts (Blestenation Mix) - Get Free Music at RCRD LBL.com]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>0</year> - <popularity>491</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Light+%26+Magic"> - <url>http://music.yahoo.com/Ladytron/Light+%26+Magic</url> - <title><![CDATA[Light & Magic]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>2002</year> - <popularity>414</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Soft+Power+Mixes"> - <url>http://music.yahoo.com/Ladytron/Soft+Power+Mixes</url> - <title><![CDATA[Soft Power Mixes]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>0</year> - <popularity>290</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Velocifero"> - <url>http://music.yahoo.com/Ladytron/Velocifero</url> - <title><![CDATA[Velocifero]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>2008</year> - <popularity>16</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ladytron/Witching+Hour"> - <url>http://music.yahoo.com/Ladytron/Witching+Hour</url> - <title><![CDATA[Witching Hour]]></title> - <artist><![CDATA[Ladytron]]></artist> - <year>2005</year> - <popularity>384</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lesbians+On+Ecstasy/Lesbians+On+Ecstasy"> - <url>http://music.yahoo.com/Lesbians+On+Ecstasy/Lesbians+On+Ecstasy</url> - <title><![CDATA[Lesbians On Ecstasy]]></title> - <artist><![CDATA[Lesbians On Ecstasy]]></artist> - <year>2004</year> - <popularity>357</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lesbians+On+Ecstasy/We+Know+You+Know"> - <url>http://music.yahoo.com/Lesbians+On+Ecstasy/We+Know+You+Know</url> - <title><![CDATA[We Know You Know]]></title> - <artist><![CDATA[Lesbians On Ecstasy]]></artist> - <year>2007</year> - <popularity>489</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madonna/Confessions+On+A+Dance+Floor"> - <url>http://music.yahoo.com/Madonna/Confessions+On+A+Dance+Floor</url> - <title><![CDATA[Confessions On A Dance Floor]]></title> - <artist><![CDATA[Madonna]]></artist> - <year>2005</year> - <popularity>190</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madonna/Hard+Candy"> - <url>http://music.yahoo.com/Madonna/Hard+Candy</url> - <title><![CDATA[Hard Candy]]></title> - <artist><![CDATA[Madonna]]></artist> - <year>2008</year> - <popularity>258</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Margaret+Berger/Chameleon"> - <url>http://music.yahoo.com/Margaret+Berger/Chameleon</url> - <title><![CDATA[Chameleon]]></title> - <artist><![CDATA[Margaret Berger]]></artist> - <year>2004</year> - <popularity>25</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Eat+Me%2C+Drink+Me"> - <url>http://music.yahoo.com/Marilyn+Manson/Eat+Me%2C+Drink+Me</url> - <title><![CDATA[Eat Me, Drink Me]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>2007</year> - <popularity>200</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Holy+Wood"> - <url>http://music.yahoo.com/Marilyn+Manson/Holy+Wood</url> - <title><![CDATA[Holy Wood]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>2000</year> - <popularity>315</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Last+Tour+On+Earth"> - <url>http://music.yahoo.com/Marilyn+Manson/Last+Tour+On+Earth</url> - <title><![CDATA[Last Tour On Earth]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>1999</year> - <popularity>480</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Last+Tour+On+Earth+%5BLimited+Edition+Bonus+Disc%5D"> - <url>http://music.yahoo.com/Marilyn+Manson/Last+Tour+On+Earth+%5BLimited+Edition+Bonus+Disc%5D</url> - <title><![CDATA[Last Tour On Earth [Limited Edition Bonus Disc]]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>0</year> - <popularity>367</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Lest+We+Forget"> - <url>http://music.yahoo.com/Marilyn+Manson/Lest+We+Forget</url> - <title><![CDATA[Lest We Forget]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>2004</year> - <popularity>5</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/Mechanical+Animals"> - <url>http://music.yahoo.com/Marilyn+Manson/Mechanical+Animals</url> - <title><![CDATA[Mechanical Animals]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>1998</year> - <popularity>12</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marilyn+Manson/The+Golden+Age+Of+Grotesque"> - <url>http://music.yahoo.com/Marilyn+Manson/The+Golden+Age+Of+Grotesque</url> - <title><![CDATA[The Golden Age Of Grotesque]]></title> - <artist><![CDATA[Marilyn Manson]]></artist> - <year>2003</year> - <popularity>343</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marsheaux/E-Bay+Queen"> - <url>http://music.yahoo.com/Marsheaux/E-Bay+Queen</url> - <title><![CDATA[E-Bay Queen]]></title> - <artist><![CDATA[Marsheaux]]></artist> - <year>2004</year> - <popularity>57</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marsheaux/Peek+a+Boo"> - <url>http://music.yahoo.com/Marsheaux/Peek+a+Boo</url> - <title><![CDATA[Peek a Boo]]></title> - <artist><![CDATA[Marsheaux]]></artist> - <year>2007</year> - <popularity>416</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Melissa+Auf+der+Maur/Auf+Der+Maur"> - <url>http://music.yahoo.com/Melissa+Auf+der+Maur/Auf+Der+Maur</url> - <title><![CDATA[Auf Der Maur]]></title> - <artist><![CDATA[Melissa Auf der Maur]]></artist> - <year>2004</year> - <popularity>313</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/_..And+Justice+For+All"> - <url>http://music.yahoo.com/Metallica/_..And+Justice+For+All</url> - <title><![CDATA[_..And Justice For All]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1988</year> - <popularity>399</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Frantic"> - <url>http://music.yahoo.com/Metallica/Frantic</url> - <title><![CDATA[Frantic]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>363</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Garage+Inc_"> - <url>http://music.yahoo.com/Metallica/Garage+Inc_</url> - <title><![CDATA[Garage Inc_]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1998</year> - <popularity>72</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Kill+%27Em+All"> - <url>http://music.yahoo.com/Metallica/Kill+%27Em+All</url> - <title><![CDATA[Kill 'Em All]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1983</year> - <popularity>405</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Master+Of+Puppets"> - <url>http://music.yahoo.com/Metallica/Master+Of+Puppets</url> - <title><![CDATA[Master Of Puppets]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1986</year> - <popularity>103</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Metallica"> - <url>http://music.yahoo.com/Metallica/Metallica</url> - <title><![CDATA[Metallica]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1991</year> - <popularity>22</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/No+Leaf+Clover"> - <url>http://music.yahoo.com/Metallica/No+Leaf+Clover</url> - <title><![CDATA[No Leaf Clover]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1999</year> - <popularity>88</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/ReLoad"> - <url>http://music.yahoo.com/Metallica/ReLoad</url> - <title><![CDATA[ReLoad]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1997</year> - <popularity>174</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Ride+The+Lightning"> - <url>http://music.yahoo.com/Metallica/Ride+The+Lightning</url> - <title><![CDATA[Ride The Lightning]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1984</year> - <popularity>344</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Some+Kind+Of+Monster"> - <url>http://music.yahoo.com/Metallica/Some+Kind+Of+Monster</url> - <title><![CDATA[Some Kind Of Monster]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>48</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/St+Anger+B+Sides"> - <url>http://music.yahoo.com/Metallica/St+Anger+B+Sides</url> - <title><![CDATA[St Anger B Sides]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>397</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/St.+Anger+Single"> - <url>http://music.yahoo.com/Metallica/St.+Anger+Single</url> - <title><![CDATA[St. Anger Single]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>372</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/The+Memory+Remains+%28Single%29"> - <url>http://music.yahoo.com/Metallica/The+Memory+Remains+%28Single%29</url> - <title><![CDATA[The Memory Remains (Single)]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1997</year> - <popularity>38</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/The+Unforgiven+II"> - <url>http://music.yahoo.com/Metallica/The+Unforgiven+II</url> - <title><![CDATA[The Unforgiven II]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1998</year> - <popularity>154</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/The+Unforgiven+II+%28Single%29"> - <url>http://music.yahoo.com/Metallica/The+Unforgiven+II+%28Single%29</url> - <title><![CDATA[The Unforgiven II (Single)]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>1998</year> - <popularity>123</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Metallica/Unnamed+Feeling+EP"> - <url>http://music.yahoo.com/Metallica/Unnamed+Feeling+EP</url> - <title><![CDATA[Unnamed Feeling EP]]></title> - <artist><![CDATA[Metallica]]></artist> - <year>2003</year> - <popularity>201</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mika/Life+In+Cartoon+Motion"> - <url>http://music.yahoo.com/Mika/Life+In+Cartoon+Motion</url> - <title><![CDATA[Life In Cartoon Motion]]></title> - <artist><![CDATA[Mika]]></artist> - <year>2006</year> - <popularity>491</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Monomen/Unknown+Album"> - <url>http://music.yahoo.com/Monomen/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Monomen]]></artist> - <year>2007</year> - <popularity>173</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Moto%CC%88rhead/Kiss+Of+Death"> - <url>http://music.yahoo.com/Moto%CC%88rhead/Kiss+Of+Death</url> - <title><![CDATA[Kiss Of Death]]></title> - <artist><![CDATA[Motörhead]]></artist> - <year>2006</year> - <popularity>479</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muse/Absolution"> - <url>http://music.yahoo.com/Muse/Absolution</url> - <title><![CDATA[Absolution]]></title> - <artist><![CDATA[Muse]]></artist> - <year>2003</year> - <popularity>487</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muse/Black+Holes+%26+Revelations"> - <url>http://music.yahoo.com/Muse/Black+Holes+%26+Revelations</url> - <title><![CDATA[Black Holes & Revelations]]></title> - <artist><![CDATA[Muse]]></artist> - <year>2006</year> - <popularity>33</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muse/Hullabaloo"> - <url>http://music.yahoo.com/Muse/Hullabaloo</url> - <title><![CDATA[Hullabaloo]]></title> - <artist><![CDATA[Muse]]></artist> - <year>2002</year> - <popularity>404</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muse/Showbiz"> - <url>http://music.yahoo.com/Muse/Showbiz</url> - <title><![CDATA[Showbiz]]></title> - <artist><![CDATA[Muse]]></artist> - <year>1999</year> - <popularity>233</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/New+Order/International"> - <url>http://music.yahoo.com/New+Order/International</url> - <title><![CDATA[International]]></title> - <artist><![CDATA[New Order]]></artist> - <year>2002</year> - <popularity>232</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Live+At+The+Royal+Albert+Hall"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/Live+At+The+Royal+Albert+Hall</url> - <title><![CDATA[Live At The Royal Albert Hall]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1998</year> - <popularity>195</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Best+Of"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Best+Of</url> - <title><![CDATA[The Best Of]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1998</year> - <popularity>170</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Boatman%27s+Call"> - <url>http://music.yahoo.com/Nick+Cave+%26+The+Bad+Seeds/The+Boatman%27s+Call</url> - <title><![CDATA[The Boatman's Call]]></title> - <artist><![CDATA[Nick Cave & The Bad Seeds]]></artist> - <year>1997</year> - <popularity>437</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nightwish/Once"> - <url>http://music.yahoo.com/Nightwish/Once</url> - <title><![CDATA[Once]]></title> - <artist><![CDATA[Nightwish]]></artist> - <year>2004</year> - <popularity>272</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nils+Petter+Molv%C3%A6r/Er"> - <url>http://music.yahoo.com/Nils+Petter+Molv%C3%A6r/Er</url> - <title><![CDATA[Er]]></title> - <artist><![CDATA[Nils Petter Molvær]]></artist> - <year>2005</year> - <popularity>10</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ozzy+Osbourne/The+Essential"> - <url>http://music.yahoo.com/Ozzy+Osbourne/The+Essential</url> - <title><![CDATA[The Essential]]></title> - <artist><![CDATA[Ozzy Osbourne]]></artist> - <year>2003</year> - <popularity>406</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ozzy+Osbourne/Tribute"> - <url>http://music.yahoo.com/Ozzy+Osbourne/Tribute</url> - <title><![CDATA[Tribute]]></title> - <artist><![CDATA[Ozzy Osbourne]]></artist> - <year>1987</year> - <popularity>367</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pantera/Reinventing+Hell"> - <url>http://music.yahoo.com/Pantera/Reinventing+Hell</url> - <title><![CDATA[Reinventing Hell]]></title> - <artist><![CDATA[Pantera]]></artist> - <year>1990</year> - <popularity>44</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pet+Shop+Boys/Disco"> - <url>http://music.yahoo.com/Pet+Shop+Boys/Disco</url> - <title><![CDATA[Disco]]></title> - <artist><![CDATA[Pet Shop Boys]]></artist> - <year>1986</year> - <popularity>119</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pet+Shop+Boys/Disco+2"> - <url>http://music.yahoo.com/Pet+Shop+Boys/Disco+2</url> - <title><![CDATA[Disco 2]]></title> - <artist><![CDATA[Pet Shop Boys]]></artist> - <year>1994</year> - <popularity>77</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pet+Shop+Boys/Discography_+The+Complete+Singles+Collection"> - <url>http://music.yahoo.com/Pet+Shop+Boys/Discography_+The+Complete+Singles+Collection</url> - <title><![CDATA[Discography_ The Complete Singles Collection]]></title> - <artist><![CDATA[Pet Shop Boys]]></artist> - <year>1985</year> - <popularity>311</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pet+Shop+Boys/Pop+Art"> - <url>http://music.yahoo.com/Pet+Shop+Boys/Pop+Art</url> - <title><![CDATA[Pop Art]]></title> - <artist><![CDATA[Pet Shop Boys]]></artist> - <year>2003</year> - <popularity>225</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Phoenix/Alphabetical"> - <url>http://music.yahoo.com/Phoenix/Alphabetical</url> - <title><![CDATA[Alphabetical]]></title> - <artist><![CDATA[Phoenix]]></artist> - <year>2004</year> - <popularity>298</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink/Try+This"> - <url>http://music.yahoo.com/Pink/Try+This</url> - <title><![CDATA[Try This]]></title> - <artist><![CDATA[Pink]]></artist> - <year>2003</year> - <popularity>246</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/Dark+Side+Of+The+Moon"> - <url>http://music.yahoo.com/Pink+Floyd/Dark+Side+Of+The+Moon</url> - <title><![CDATA[Dark Side Of The Moon]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>1973</year> - <popularity>188</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/The+Wall"> - <url>http://music.yahoo.com/Pink+Floyd/The+Wall</url> - <title><![CDATA[The Wall]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>1979</year> - <popularity>494</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pixies/Wave+Of+Mutilation_+Best+Of+Pixies"> - <url>http://music.yahoo.com/Pixies/Wave+Of+Mutilation_+Best+Of+Pixies</url> - <title><![CDATA[Wave Of Mutilation_ Best Of Pixies]]></title> - <artist><![CDATA[Pixies]]></artist> - <year>1988</year> - <popularity>7</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queen/Greatest+Hits+I"> - <url>http://music.yahoo.com/Queen/Greatest+Hits+I</url> - <title><![CDATA[Greatest Hits I]]></title> - <artist><![CDATA[Queen]]></artist> - <year>1975</year> - <popularity>421</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queen/Innuendo"> - <url>http://music.yahoo.com/Queen/Innuendo</url> - <title><![CDATA[Innuendo]]></title> - <artist><![CDATA[Queen]]></artist> - <year>1991</year> - <popularity>28</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queensry%CC%88che/Empire"> - <url>http://music.yahoo.com/Queensry%CC%88che/Empire</url> - <title><![CDATA[Empire]]></title> - <artist><![CDATA[Queensrÿche]]></artist> - <year>1990</year> - <popularity>52</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queensry%CC%88che/Operation_+Mindcrime"> - <url>http://music.yahoo.com/Queensry%CC%88che/Operation_+Mindcrime</url> - <title><![CDATA[Operation_ Mindcrime]]></title> - <artist><![CDATA[Queensrÿche]]></artist> - <year>1988</year> - <popularity>210</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Queensry%CC%88che/Rage+For+Order"> - <url>http://music.yahoo.com/Queensry%CC%88che/Rage+For+Order</url> - <title><![CDATA[Rage For Order]]></title> - <artist><![CDATA[Queensrÿche]]></artist> - <year>1986</year> - <popularity>290</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/R.E.M_/Out+Of+Time"> - <url>http://music.yahoo.com/R.E.M_/Out+Of+Time</url> - <title><![CDATA[Out Of Time]]></title> - <artist><![CDATA[R.E.M_]]></artist> - <year>0</year> - <popularity>75</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/R.E.M_/The+Best+Of+R.E.M_"> - <url>http://music.yahoo.com/R.E.M_/The+Best+Of+R.E.M_</url> - <title><![CDATA[The Best Of R.E.M_]]></title> - <artist><![CDATA[R.E.M_]]></artist> - <year>1982</year> - <popularity>180</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Radiohead/The+Bends"> - <url>http://music.yahoo.com/Radiohead/The+Bends</url> - <title><![CDATA[The Bends]]></title> - <artist><![CDATA[Radiohead]]></artist> - <year>1995</year> - <popularity>119</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/Raga+Live"> - <url>http://music.yahoo.com/Raga+Rockers/Raga+Live</url> - <title><![CDATA[Raga Live]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>1990</year> - <popularity>282</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/The+Beginning+Of+The+Raga+Rockers"> - <url>http://music.yahoo.com/Raga+Rockers/The+Beginning+Of+The+Raga+Rockers</url> - <title><![CDATA[The Beginning Of The Raga Rockers]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>1989</year> - <popularity>290</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/U%CC%88bermensch"> - <url>http://music.yahoo.com/Raga+Rockers/U%CC%88bermensch</url> - <title><![CDATA[Übermensch]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>2007</year> - <popularity>422</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Herzeleid"> - <url>http://music.yahoo.com/Rammstein/Herzeleid</url> - <title><![CDATA[Herzeleid]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>1995</year> - <popularity>264</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Keine+Lust+%5BGerman+Single+%232%5D"> - <url>http://music.yahoo.com/Rammstein/Keine+Lust+%5BGerman+Single+%232%5D</url> - <title><![CDATA[Keine Lust [German Single #2]]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>0</year> - <popularity>351</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Mutter"> - <url>http://music.yahoo.com/Rammstein/Mutter</url> - <title><![CDATA[Mutter]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>2001</year> - <popularity>283</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Reise%2C+Reise"> - <url>http://music.yahoo.com/Rammstein/Reise%2C+Reise</url> - <title><![CDATA[Reise, Reise]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>2004</year> - <popularity>376</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Rosenrot"> - <url>http://music.yahoo.com/Rammstein/Rosenrot</url> - <title><![CDATA[Rosenrot]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>2005</year> - <popularity>492</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Sehnsucht"> - <url>http://music.yahoo.com/Rammstein/Sehnsucht</url> - <title><![CDATA[Sehnsucht]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>1997</year> - <popularity>430</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rammstein/Vo%CC%88lkerball"> - <url>http://music.yahoo.com/Rammstein/Vo%CC%88lkerball</url> - <title><![CDATA[Völkerball]]></title> - <artist><![CDATA[Rammstein]]></artist> - <year>2006</year> - <popularity>169</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Red+Hot+Chili+Peppers/Blood+Sugar+Sex+Magik"> - <url>http://music.yahoo.com/Red+Hot+Chili+Peppers/Blood+Sugar+Sex+Magik</url> - <title><![CDATA[Blood Sugar Sex Magik]]></title> - <artist><![CDATA[Red Hot Chili Peppers]]></artist> - <year>1991</year> - <popularity>277</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Red+Hot+Chili+Peppers/Give+It+Away+%5BSingle%5D"> - <url>http://music.yahoo.com/Red+Hot+Chili+Peppers/Give+It+Away+%5BSingle%5D</url> - <title><![CDATA[Give It Away [Single]]]></title> - <artist><![CDATA[Red Hot Chili Peppers]]></artist> - <year>0</year> - <popularity>427</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Red+Hot+Chili+Peppers/Stadium+Arcadium"> - <url>http://music.yahoo.com/Red+Hot+Chili+Peppers/Stadium+Arcadium</url> - <title><![CDATA[Stadium Arcadium]]></title> - <artist><![CDATA[Red Hot Chili Peppers]]></artist> - <year>0</year> - <popularity>364</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Roxy+Music/Avalon"> - <url>http://music.yahoo.com/Roxy+Music/Avalon</url> - <title><![CDATA[Avalon]]></title> - <artist><![CDATA[Roxy Music]]></artist> - <year>1982</year> - <popularity>142</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Roxy+Music/Love+Is+The+Drug+%28single%29"> - <url>http://music.yahoo.com/Roxy+Music/Love+Is+The+Drug+%28single%29</url> - <title><![CDATA[Love Is The Drug (single)]]></title> - <artist><![CDATA[Roxy Music]]></artist> - <year>1990</year> - <popularity>176</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ro%CC%88yksopp/The+Understanding"> - <url>http://music.yahoo.com/Ro%CC%88yksopp/The+Understanding</url> - <title><![CDATA[The Understanding]]></title> - <artist><![CDATA[Röyksopp]]></artist> - <year>2005</year> - <popularity>337</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rustie+%26+Hudson+Mohawke/Unknown+Album"> - <url>http://music.yahoo.com/Rustie+%26+Hudson+Mohawke/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[Rustie & Hudson Mohawke]]></artist> - <year>0</year> - <popularity>157</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Satyricon/Now%2C+Diabolical"> - <url>http://music.yahoo.com/Satyricon/Now%2C+Diabolical</url> - <title><![CDATA[Now, Diabolical]]></title> - <artist><![CDATA[Satyricon]]></artist> - <year>2006</year> - <popularity>366</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Scissor+Sisters/Ta-Dah%21"> - <url>http://music.yahoo.com/Scissor+Sisters/Ta-Dah%21</url> - <title><![CDATA[Ta-Dah!]]></title> - <artist><![CDATA[Scissor Sisters]]></artist> - <year>2006</year> - <popularity>308</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Scorpions/Scorpions+Best"> - <url>http://music.yahoo.com/Scorpions/Scorpions+Best</url> - <title><![CDATA[Scorpions Best]]></title> - <artist><![CDATA[Scorpions]]></artist> - <year>1999</year> - <popularity>467</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Shakespears+Sister/Hormonally+Yours"> - <url>http://music.yahoo.com/Shakespears+Sister/Hormonally+Yours</url> - <title><![CDATA[Hormonally Yours]]></title> - <artist><![CDATA[Shakespears Sister]]></artist> - <year>1992</year> - <popularity>71</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sheryl+Crow/Sheryl+Crow"> - <url>http://music.yahoo.com/Sheryl+Crow/Sheryl+Crow</url> - <title><![CDATA[Sheryl Crow]]></title> - <artist><![CDATA[Sheryl Crow]]></artist> - <year>1996</year> - <popularity>295</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Shy+Child/Drop+the+Phone+-+Single"> - <url>http://music.yahoo.com/Shy+Child/Drop+the+Phone+-+Single</url> - <title><![CDATA[Drop the Phone - Single]]></title> - <artist><![CDATA[Shy Child]]></artist> - <year>2007</year> - <popularity>328</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Slayer/Christ+Illusion"> - <url>http://music.yahoo.com/Slayer/Christ+Illusion</url> - <title><![CDATA[Christ Illusion]]></title> - <artist><![CDATA[Slayer]]></artist> - <year>2006</year> - <popularity>402</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Smashing+Pumpkins/Mellon+Collie+And+The+Infinite+Sadness"> - <url>http://music.yahoo.com/Smashing+Pumpkins/Mellon+Collie+And+The+Infinite+Sadness</url> - <title><![CDATA[Mellon Collie And The Infinite Sadness]]></title> - <artist><![CDATA[Smashing Pumpkins]]></artist> - <year>1995</year> - <popularity>358</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Suede/Coming+Up"> - <url>http://music.yahoo.com/Suede/Coming+Up</url> - <title><![CDATA[Coming Up]]></title> - <artist><![CDATA[Suede]]></artist> - <year>1996</year> - <popularity>263</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ten+Benson/Danger+Of+Deaf"> - <url>http://music.yahoo.com/Ten+Benson/Danger+Of+Deaf</url> - <title><![CDATA[Danger Of Deaf]]></title> - <artist><![CDATA[Ten Benson]]></artist> - <year>2003</year> - <popularity>71</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ten+Benson/Hiss"> - <url>http://music.yahoo.com/Ten+Benson/Hiss</url> - <title><![CDATA[Hiss]]></title> - <artist><![CDATA[Ten Benson]]></artist> - <year>2000</year> - <popularity>335</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ten+Benson/Satan+Kidney+Pie"> - <url>http://music.yahoo.com/Ten+Benson/Satan+Kidney+Pie</url> - <title><![CDATA[Satan Kidney Pie]]></title> - <artist><![CDATA[Ten Benson]]></artist> - <year>0</year> - <popularity>193</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/A+Forest_+The+Singles"> - <url>http://music.yahoo.com/The+Cure/A+Forest_+The+Singles</url> - <title><![CDATA[A Forest_ The Singles]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>359</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/A+Letter+To+Elise"> - <url>http://music.yahoo.com/The+Cure/A+Letter+To+Elise</url> - <title><![CDATA[A Letter To Elise]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1992</year> - <popularity>89</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Bloodflowers"> - <url>http://music.yahoo.com/The+Cure/Bloodflowers</url> - <title><![CDATA[Bloodflowers]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>2000</year> - <popularity>148</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Boys+Don%27t+Cry"> - <url>http://music.yahoo.com/The+Cure/Boys+Don%27t+Cry</url> - <title><![CDATA[Boys Don't Cry]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1979</year> - <popularity>144</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Close+To+Me+%28Single%29"> - <url>http://music.yahoo.com/The+Cure/Close+To+Me+%28Single%29</url> - <title><![CDATA[Close To Me (Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>357</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Concert_+The+Cure+Live"> - <url>http://music.yahoo.com/The+Cure/Concert_+The+Cure+Live</url> - <title><![CDATA[Concert_ The Cure Live]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1984</year> - <popularity>491</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Cut+Here+%28Single%29"> - <url>http://music.yahoo.com/The+Cure/Cut+Here+%28Single%29</url> - <title><![CDATA[Cut Here (Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>2001</year> - <popularity>19</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Disintegration"> - <url>http://music.yahoo.com/The+Cure/Disintegration</url> - <title><![CDATA[Disintegration]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1989</year> - <popularity>180</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Entreat"> - <url>http://music.yahoo.com/The+Cure/Entreat</url> - <title><![CDATA[Entreat]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Faith"> - <url>http://music.yahoo.com/The+Cure/Faith</url> - <title><![CDATA[Faith]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1981</year> - <popularity>217</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Faith_+Rarities+1980-1981"> - <url>http://music.yahoo.com/The+Cure/Faith_+Rarities+1980-1981</url> - <title><![CDATA[Faith_ Rarities 1980-1981]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1980</year> - <popularity>412</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Five+Swing+Live"> - <url>http://music.yahoo.com/The+Cure/Five+Swing+Live</url> - <title><![CDATA[Five Swing Live]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1997</year> - <popularity>280</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Friday+I%27m+In+Love+%28Single%29"> - <url>http://music.yahoo.com/The+Cure/Friday+I%27m+In+Love+%28Single%29</url> - <title><![CDATA[Friday I'm In Love (Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1992</year> - <popularity>64</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Gone%21"> - <url>http://music.yahoo.com/The+Cure/Gone%21</url> - <title><![CDATA[Gone!]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1996</year> - <popularity>204</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Head+On+The+Door+-+Rarities+1984-1985"> - <url>http://music.yahoo.com/The+Cure/Head+On+The+Door+-+Rarities+1984-1985</url> - <title><![CDATA[Head On The Door - Rarities 1984-1985]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1984</year> - <popularity>309</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/High+%28Mini+Single%29"> - <url>http://music.yahoo.com/The+Cure/High+%28Mini+Single%29</url> - <title><![CDATA[High (Mini Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1992</year> - <popularity>148</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/High+%28Single%29"> - <url>http://music.yahoo.com/The+Cure/High+%28Single%29</url> - <title><![CDATA[High (Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1992</year> - <popularity>263</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Japanese+Whispers"> - <url>http://music.yahoo.com/The+Cure/Japanese+Whispers</url> - <title><![CDATA[Japanese Whispers]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1982</year> - <popularity>219</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Join+The+Dots_+B-Sides+%26+Rarities+1978-2001"> - <url>http://music.yahoo.com/The+Cure/Join+The+Dots_+B-Sides+%26+Rarities+1978-2001</url> - <title><![CDATA[Join The Dots_ B-Sides & Rarities 1978-2001]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>2004</year> - <popularity>87</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Kiss+Me+Kiss+Me+Kiss+Me"> - <url>http://music.yahoo.com/The+Cure/Kiss+Me+Kiss+Me+Kiss+Me</url> - <title><![CDATA[Kiss Me Kiss Me Kiss Me]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1987</year> - <popularity>20</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Kiss+Me+Kiss+Me+Kiss+Me_+Rarities+1986+%E2%80%931987"> - <url>http://music.yahoo.com/The+Cure/Kiss+Me+Kiss+Me+Kiss+Me_+Rarities+1986+%E2%80%931987</url> - <title><![CDATA[Kiss Me Kiss Me Kiss Me_ Rarities 1986 –1987]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1987</year> - <popularity>227</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Lovesong"> - <url>http://music.yahoo.com/The+Cure/Lovesong</url> - <title><![CDATA[Lovesong]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1989</year> - <popularity>408</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Mint+Car"> - <url>http://music.yahoo.com/The+Cure/Mint+Car</url> - <title><![CDATA[Mint Car]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1996</year> - <popularity>361</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Never+Enough+%28Single%29"> - <url>http://music.yahoo.com/The+Cure/Never+Enough+%28Single%29</url> - <title><![CDATA[Never Enough (Single)]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>219</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Never+Enough+-+Bootleg+Consert"> - <url>http://music.yahoo.com/The+Cure/Never+Enough+-+Bootleg+Consert</url> - <title><![CDATA[Never Enough - Bootleg Consert]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>122</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Pictures+Of+You"> - <url>http://music.yahoo.com/The+Cure/Pictures+Of+You</url> - <title><![CDATA[Pictures Of You]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1990</year> - <popularity>409</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Pornography"> - <url>http://music.yahoo.com/The+Cure/Pornography</url> - <title><![CDATA[Pornography]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1982</year> - <popularity>350</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Pornography_+Rarities+1981-1982"> - <url>http://music.yahoo.com/The+Cure/Pornography_+Rarities+1981-1982</url> - <title><![CDATA[Pornography_ Rarities 1981-1982]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1981</year> - <popularity>341</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Seventeen+Seconds"> - <url>http://music.yahoo.com/The+Cure/Seventeen+Seconds</url> - <title><![CDATA[Seventeen Seconds]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1980</year> - <popularity>5</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Seventeen+Seconds...+_+Rarities+1979-1980"> - <url>http://music.yahoo.com/The+Cure/Seventeen+Seconds...+_+Rarities+1979-1980</url> - <title><![CDATA[Seventeen Seconds... _ Rarities 1979-1980]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>2005</year> - <popularity>422</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Sideshow"> - <url>http://music.yahoo.com/The+Cure/Sideshow</url> - <title><![CDATA[Sideshow]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1993</year> - <popularity>369</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Strange+Attraction"> - <url>http://music.yahoo.com/The+Cure/Strange+Attraction</url> - <title><![CDATA[Strange Attraction]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1996</year> - <popularity>227</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+13th"> - <url>http://music.yahoo.com/The+Cure/The+13th</url> - <title><![CDATA[The 13th]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1996</year> - <popularity>361</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+Cure"> - <url>http://music.yahoo.com/The+Cure/The+Cure</url> - <title><![CDATA[The Cure]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>2004</year> - <popularity>102</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+Head+On+The+Door"> - <url>http://music.yahoo.com/The+Cure/The+Head+On+The+Door</url> - <title><![CDATA[The Head On The Door]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1985</year> - <popularity>328</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+Peel+Sessions"> - <url>http://music.yahoo.com/The+Cure/The+Peel+Sessions</url> - <title><![CDATA[The Peel Sessions]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1978</year> - <popularity>46</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+Top"> - <url>http://music.yahoo.com/The+Cure/The+Top</url> - <title><![CDATA[The Top]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1984</year> - <popularity>437</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/The+Top_+Rarities+1982-1984"> - <url>http://music.yahoo.com/The+Cure/The+Top_+Rarities+1982-1984</url> - <title><![CDATA[The Top_ Rarities 1982-1984]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1984</year> - <popularity>361</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Three+Imaginary+Boys"> - <url>http://music.yahoo.com/The+Cure/Three+Imaginary+Boys</url> - <title><![CDATA[Three Imaginary Boys]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1979</year> - <popularity>297</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Three+Imaginary+Boys_+Rarities+1977-1979"> - <url>http://music.yahoo.com/The+Cure/Three+Imaginary+Boys_+Rarities+1977-1979</url> - <title><![CDATA[Three Imaginary Boys_ Rarities 1977-1979]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1979</year> - <popularity>104</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Wrong+Number"> - <url>http://music.yahoo.com/The+Cure/Wrong+Number</url> - <title><![CDATA[Wrong Number]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>1997</year> - <popularity>224</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Killers/Hot+Fuss"> - <url>http://music.yahoo.com/The+Killers/Hot+Fuss</url> - <title><![CDATA[Hot Fuss]]></title> - <artist><![CDATA[The Killers]]></artist> - <year>2004</year> - <popularity>375</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Knife/Deep+Cuts"> - <url>http://music.yahoo.com/The+Knife/Deep+Cuts</url> - <title><![CDATA[Deep Cuts]]></title> - <artist><![CDATA[The Knife]]></artist> - <year>2006</year> - <popularity>469</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Knife/Like+A+Pen+%28Single%29"> - <url>http://music.yahoo.com/The+Knife/Like+A+Pen+%28Single%29</url> - <title><![CDATA[Like A Pen (Single)]]></title> - <artist><![CDATA[The Knife]]></artist> - <year>2006</year> - <popularity>106</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Knife/Silent+Shout"> - <url>http://music.yahoo.com/The+Knife/Silent+Shout</url> - <title><![CDATA[Silent Shout]]></title> - <artist><![CDATA[The Knife]]></artist> - <year>2006</year> - <popularity>419</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Knife/The+Knife"> - <url>http://music.yahoo.com/The+Knife/The+Knife</url> - <title><![CDATA[The Knife]]></title> - <artist><![CDATA[The Knife]]></artist> - <year>2005</year> - <popularity>149</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Knife/We+Share+Our+Mothers%27+Health"> - <url>http://music.yahoo.com/The+Knife/We+Share+Our+Mothers%27+Health</url> - <title><![CDATA[We Share Our Mothers' Health]]></title> - <artist><![CDATA[The Knife]]></artist> - <year>2006</year> - <popularity>133</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Pussycat+Dolls/Don%27t+Cha+-+Single"> - <url>http://music.yahoo.com/The+Pussycat+Dolls/Don%27t+Cha+-+Single</url> - <title><![CDATA[Don't Cha - Single]]></title> - <artist><![CDATA[The Pussycat Dolls]]></artist> - <year>2005</year> - <popularity>183</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Sinister+Sound+Syndicate/Unknown+Album"> - <url>http://music.yahoo.com/The+Sinister+Sound+Syndicate/Unknown+Album</url> - <title><![CDATA[Unknown Album]]></title> - <artist><![CDATA[The Sinister Sound Syndicate]]></artist> - <year>2008</year> - <popularity>75</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Timbuktu/Tack+For+Kaffet"> - <url>http://music.yahoo.com/Timbuktu/Tack+For+Kaffet</url> - <title><![CDATA[Tack For Kaffet]]></title> - <artist><![CDATA[Timbuktu]]></artist> - <year>0</year> - <popularity>123</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Van+Halen/The+Best+Of+Van+Halen%2C+Vol.+I"> - <url>http://music.yahoo.com/Van+Halen/The+Best+Of+Van+Halen%2C+Vol.+I</url> - <title><![CDATA[The Best Of Van Halen, Vol. I]]></title> - <artist><![CDATA[Van Halen]]></artist> - <year>1978</year> - <popularity>277</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Wamdue+Project/King+Of+My+Castle"> - <url>http://music.yahoo.com/Wamdue+Project/King+Of+My+Castle</url> - <title><![CDATA[King Of My Castle]]></title> - <artist><![CDATA[Wamdue Project]]></artist> - <year>1998</year> - <popularity>310</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Wolfmother/Wolfmother"> - <url>http://music.yahoo.com/Wolfmother/Wolfmother</url> - <title><![CDATA[Wolfmother]]></title> - <artist><![CDATA[Wolfmother]]></artist> - <year>2006</year> - <popularity>404</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Yelle/Kitsune%CC%81_+A%CC%80+cause+des+garc%CC%A7ons+Remixes+-+EP"> - <url>http://music.yahoo.com/Yelle/Kitsune%CC%81_+A%CC%80+cause+des+garc%CC%A7ons+Remixes+-+EP</url> - <title><![CDATA[Kitsuné_ À cause des garçons Remixes - EP]]></title> - <artist><![CDATA[Yelle]]></artist> - <year>2007</year> - <popularity>279</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Yelle/Pop+Up+-+Version+deluxe"> - <url>http://music.yahoo.com/Yelle/Pop+Up+-+Version+deluxe</url> - <title><![CDATA[Pop Up - Version deluxe]]></title> - <artist><![CDATA[Yelle]]></artist> - <year>2007</year> - <popularity>499</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Afro+Cuban+All-Stars/A+Toda+Cuba+Le+Gusta"> - <url>http://music.yahoo.com/Afro+Cuban+All-Stars/A+Toda+Cuba+Le+Gusta</url> - <title><![CDATA[A Toda Cuba Le Gusta]]></title> - <artist><![CDATA[Afro Cuban All-Stars]]></artist> - <year>0</year> - <popularity>22</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Afro+Cuban+All-Stars/Distinto%2C+Diferente"> - <url>http://music.yahoo.com/Afro+Cuban+All-Stars/Distinto%2C+Diferente</url> - <title><![CDATA[Distinto, Diferente]]></title> - <artist><![CDATA[Afro Cuban All-Stars]]></artist> - <year>0</year> - <popularity>331</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Agnes+Buen+Garna%CC%8As+%26+Jan+Garbarek/Rosensfole"> - <url>http://music.yahoo.com/Agnes+Buen+Garna%CC%8As+%26+Jan+Garbarek/Rosensfole</url> - <title><![CDATA[Rosensfole]]></title> - <artist><![CDATA[Agnes Buen Garnås & Jan Garbarek]]></artist> - <year>0</year> - <popularity>391</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Arve+Tellefsen/Pan"> - <url>http://music.yahoo.com/Arve+Tellefsen/Pan</url> - <title><![CDATA[Pan]]></title> - <artist><![CDATA[Arve Tellefsen]]></artist> - <year>0</year> - <popularity>447</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Astrud+Gilberto/Beach+Samba"> - <url>http://music.yahoo.com/Astrud+Gilberto/Beach+Samba</url> - <title><![CDATA[Beach Samba]]></title> - <artist><![CDATA[Astrud Gilberto]]></artist> - <year>0</year> - <popularity>496</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Berge/Live+in+hell"> - <url>http://music.yahoo.com/Berge/Live+in+hell</url> - <title><![CDATA[Live in hell]]></title> - <artist><![CDATA[Berge]]></artist> - <year>0</year> - <popularity>99</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bjo%CC%88rk/Debut"> - <url>http://music.yahoo.com/Bjo%CC%88rk/Debut</url> - <title><![CDATA[Debut]]></title> - <artist><![CDATA[Björk]]></artist> - <year>0</year> - <popularity>430</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bjo%CC%88rk/Homogenic"> - <url>http://music.yahoo.com/Bjo%CC%88rk/Homogenic</url> - <title><![CDATA[Homogenic]]></title> - <artist><![CDATA[Björk]]></artist> - <year>0</year> - <popularity>146</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bj%C3%B8rn+Berge/Bag+Of+Nails"> - <url>http://music.yahoo.com/Bj%C3%B8rn+Berge/Bag+Of+Nails</url> - <title><![CDATA[Bag Of Nails]]></title> - <artist><![CDATA[Bjørn Berge]]></artist> - <year>0</year> - <popularity>456</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bj%C3%B8rn+Berge/Bj%C3%B8rn+Berge"> - <url>http://music.yahoo.com/Bj%C3%B8rn+Berge/Bj%C3%B8rn+Berge</url> - <title><![CDATA[Bjørn Berge]]></title> - <artist><![CDATA[Bjørn Berge]]></artist> - <year>0</year> - <popularity>479</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bj%C3%B8rn+Berge/Blues+Hit+Me"> - <url>http://music.yahoo.com/Bj%C3%B8rn+Berge/Blues+Hit+Me</url> - <title><![CDATA[Blues Hit Me]]></title> - <artist><![CDATA[Bjørn Berge]]></artist> - <year>0</year> - <popularity>364</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bj%C3%B8rn+Berge/Illustrated+Man"> - <url>http://music.yahoo.com/Bj%C3%B8rn+Berge/Illustrated+Man</url> - <title><![CDATA[Illustrated Man]]></title> - <artist><![CDATA[Bjørn Berge]]></artist> - <year>0</year> - <popularity>188</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bj%C3%B8rn+Eidsva%CC%8Ag/Alt+Du+Vil+Ha"> - <url>http://music.yahoo.com/Bj%C3%B8rn+Eidsva%CC%8Ag/Alt+Du+Vil+Ha</url> - <title><![CDATA[Alt Du Vil Ha]]></title> - <artist><![CDATA[Bjørn Eidsvåg]]></artist> - <year>0</year> - <popularity>365</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Blind-Dog+Gatewood/I+Hear+the+Blues+Callin%27+and+the+Devil+Is+Near+%21"> - <url>http://music.yahoo.com/Blind-Dog+Gatewood/I+Hear+the+Blues+Callin%27+and+the+Devil+Is+Near+%21</url> - <title><![CDATA[I Hear the Blues Callin' and the Devil Is Near !]]></title> - <artist><![CDATA[Blind-Dog Gatewood]]></artist> - <year>0</year> - <popularity>449</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bo+Kaspers+Orkester/Pa%CC%8A+Hotell"> - <url>http://music.yahoo.com/Bo+Kaspers+Orkester/Pa%CC%8A+Hotell</url> - <title><![CDATA[På Hotell]]></title> - <artist><![CDATA[Bo Kaspers Orkester]]></artist> - <year>0</year> - <popularity>233</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Dylan/Bob+Dylan+%28Remastered+Promotional+Super+Audio+CD%29"> - <url>http://music.yahoo.com/Bob+Dylan/Bob+Dylan+%28Remastered+Promotional+Super+Audio+CD%29</url> - <title><![CDATA[Bob Dylan (Remastered Promotional Super Audio CD)]]></title> - <artist><![CDATA[Bob Dylan]]></artist> - <year>0</year> - <popularity>264</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Marley/The+Legends+Collection"> - <url>http://music.yahoo.com/Bob+Marley/The+Legends+Collection</url> - <title><![CDATA[The Legends Collection]]></title> - <artist><![CDATA[Bob Marley]]></artist> - <year>0</year> - <popularity>364</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Marley+%26+The+Wailers/3+Lions"> - <url>http://music.yahoo.com/Bob+Marley+%26+The+Wailers/3+Lions</url> - <title><![CDATA[3 Lions]]></title> - <artist><![CDATA[Bob Marley & The Wailers]]></artist> - <year>0</year> - <popularity>207</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bob+Marley+%26+The+Wailers/Legend"> - <url>http://music.yahoo.com/Bob+Marley+%26+The+Wailers/Legend</url> - <title><![CDATA[Legend]]></title> - <artist><![CDATA[Bob Marley & The Wailers]]></artist> - <year>0</year> - <popularity>135</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bruce+Springsteen/Human+Touch"> - <url>http://music.yahoo.com/Bruce+Springsteen/Human+Touch</url> - <title><![CDATA[Human Touch]]></title> - <artist><![CDATA[Bruce Springsteen]]></artist> - <year>0</year> - <popularity>254</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bruce+Springsteen/Human+Touch+%28CD5+Single+EP%29"> - <url>http://music.yahoo.com/Bruce+Springsteen/Human+Touch+%28CD5+Single+EP%29</url> - <title><![CDATA[Human Touch (CD5 Single EP)]]></title> - <artist><![CDATA[Bruce Springsteen]]></artist> - <year>0</year> - <popularity>226</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bruce+Springsteen/Live+1975+-+1985"> - <url>http://music.yahoo.com/Bruce+Springsteen/Live+1975+-+1985</url> - <title><![CDATA[Live 1975 - 1985]]></title> - <artist><![CDATA[Bruce Springsteen]]></artist> - <year>0</year> - <popularity>66</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bruce+Springsteen/The+River"> - <url>http://music.yahoo.com/Bruce+Springsteen/The+River</url> - <title><![CDATA[The River]]></title> - <artist><![CDATA[Bruce Springsteen]]></artist> - <year>0</year> - <popularity>431</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Bryan+Ferry/Another+Time%2C+Another+Place"> - <url>http://music.yahoo.com/Bryan+Ferry/Another+Time%2C+Another+Place</url> - <title><![CDATA[Another Time, Another Place]]></title> - <artist><![CDATA[Bryan Ferry]]></artist> - <year>0</year> - <popularity>483</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Chris+Isaak/Wicked+Game"> - <url>http://music.yahoo.com/Chris+Isaak/Wicked+Game</url> - <title><![CDATA[Wicked Game]]></title> - <artist><![CDATA[Chris Isaak]]></artist> - <year>0</year> - <popularity>136</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Cirque+Du+Soleil/Alegri%CC%81a"> - <url>http://music.yahoo.com/Cirque+Du+Soleil/Alegri%CC%81a</url> - <title><![CDATA[Alegría]]></title> - <artist><![CDATA[Cirque Du Soleil]]></artist> - <year>0</year> - <popularity>45</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Cirque+Du+Soleil/Saltimbanco"> - <url>http://music.yahoo.com/Cirque+Du+Soleil/Saltimbanco</url> - <title><![CDATA[Saltimbanco]]></title> - <artist><![CDATA[Cirque Du Soleil]]></artist> - <year>0</year> - <popularity>133</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compay+Segundo/Lo+Mejor+De+La+Vida"> - <url>http://music.yahoo.com/Compay+Segundo/Lo+Mejor+De+La+Vida</url> - <title><![CDATA[Lo Mejor De La Vida]]></title> - <artist><![CDATA[Compay Segundo]]></artist> - <year>0</year> - <popularity>77</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/A+Buena+Vista_+Barrio+de+la+Habana"> - <url>http://music.yahoo.com/Compilations/A+Buena+Vista_+Barrio+de+la+Habana</url> - <title><![CDATA[A Buena Vista_ Barrio de la Habana]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>4</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Acoustic+Blues"> - <url>http://music.yahoo.com/Compilations/Acoustic+Blues</url> - <title><![CDATA[Acoustic Blues]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>91</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Big"> - <url>http://music.yahoo.com/Compilations/Big</url> - <title><![CDATA[Big]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>439</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Brazilian+Portrait_+Villa-Lobos+And+The+Guitar+Music+Of+Brazil"> - <url>http://music.yahoo.com/Compilations/Brazilian+Portrait_+Villa-Lobos+And+The+Guitar+Music+Of+Brazil</url> - <title><![CDATA[Brazilian Portrait_ Villa-Lobos And The Guitar Music Of Brazil]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>478</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Cornelis+Ba%CC%88sta"> - <url>http://music.yahoo.com/Compilations/Cornelis+Ba%CC%88sta</url> - <title><![CDATA[Cornelis Bästa]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>165</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Dimples+%28The+best+of+John+Lee+Hooker%29"> - <url>http://music.yahoo.com/Compilations/Dimples+%28The+best+of+John+Lee+Hooker%29</url> - <title><![CDATA[Dimples (The best of John Lee Hooker)]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>309</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/El+Sonido+De+Flamenco"> - <url>http://music.yahoo.com/Compilations/El+Sonido+De+Flamenco</url> - <title><![CDATA[El Sonido De Flamenco]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>257</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Gold+Ballads"> - <url>http://music.yahoo.com/Compilations/Gold+Ballads</url> - <title><![CDATA[Gold Ballads]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>50</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Heart+Still+Beating+%5BLive%5D"> - <url>http://music.yahoo.com/Compilations/Heart+Still+Beating+%5BLive%5D</url> - <title><![CDATA[Heart Still Beating [Live]]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>277</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Himmelskip"> - <url>http://music.yahoo.com/Compilations/Himmelskip</url> - <title><![CDATA[Himmelskip]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>117</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/L.+A.+Woman"> - <url>http://music.yahoo.com/Compilations/L.+A.+Woman</url> - <title><![CDATA[L. A. Woman]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>397</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Leadbelly+-+Definitive+Leadbelly"> - <url>http://music.yahoo.com/Compilations/Leadbelly+-+Definitive+Leadbelly</url> - <title><![CDATA[Leadbelly - Definitive Leadbelly]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>57</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Made+In+Havana"> - <url>http://music.yahoo.com/Compilations/Made+In+Havana</url> - <title><![CDATA[Made In Havana]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>499</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/MTV+Unplugged+%5BLive%5D"> - <url>http://music.yahoo.com/Compilations/MTV+Unplugged+%5BLive%5D</url> - <title><![CDATA[MTV Unplugged [Live]]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>291</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Steppenwolf+Gold+Their+Greatest+Hits"> - <url>http://music.yahoo.com/Compilations/Steppenwolf+Gold+Their+Greatest+Hits</url> - <title><![CDATA[Steppenwolf Gold Their Greatest Hits]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>62</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Story+of+the+Blues"> - <url>http://music.yahoo.com/Compilations/Story+of+the+Blues</url> - <title><![CDATA[Story of the Blues]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>293</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/SuperHarps+II"> - <url>http://music.yahoo.com/Compilations/SuperHarps+II</url> - <title><![CDATA[SuperHarps II]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>444</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Best+Cuban+Album+in+the+World+...+Ever%21"> - <url>http://music.yahoo.com/Compilations/The+Best+Cuban+Album+in+the+World+...+Ever%21</url> - <title><![CDATA[The Best Cuban Album in the World ... Ever!]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>232</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Lord+Of+The+Rings_+The+Fellowship+Of+The+Ring"> - <url>http://music.yahoo.com/Compilations/The+Lord+Of+The+Rings_+The+Fellowship+Of+The+Ring</url> - <title><![CDATA[The Lord Of The Rings_ The Fellowship Of The Ring]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>372</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Mystery+of+Santo+Domingo+de+Silos_+Gregorian+Chant+from+Spain"> - <url>http://music.yahoo.com/Compilations/The+Mystery+of+Santo+Domingo+de+Silos_+Gregorian+Chant+from+Spain</url> - <title><![CDATA[The Mystery of Santo Domingo de Silos_ Gregorian Chant from Spain]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>173</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Rest+Of+The+Best"> - <url>http://music.yahoo.com/Compilations/The+Rest+Of+The+Best</url> - <title><![CDATA[The Rest Of The Best]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>316</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Secret+Life+Of+The+Waterboys"> - <url>http://music.yahoo.com/Compilations/The+Secret+Life+Of+The+Waterboys</url> - <title><![CDATA[The Secret Life Of The Waterboys]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>490</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/The+Three+Tenors+In+Concert"> - <url>http://music.yahoo.com/Compilations/The+Three+Tenors+In+Concert</url> - <title><![CDATA[The Three Tenors In Concert]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>120</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Compilations/Wild+At+Heart"> - <url>http://music.yahoo.com/Compilations/Wild+At+Heart</url> - <title><![CDATA[Wild At Heart]]></title> - <artist><![CDATA[Compilations]]></artist> - <year>0</year> - <popularity>244</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Cornelis+Vreeswijk/Guldkorn+Fra%CC%8An+Ma%CC%88ster+Cees+Memoarer"> - <url>http://music.yahoo.com/Cornelis+Vreeswijk/Guldkorn+Fra%CC%8An+Ma%CC%88ster+Cees+Memoarer</url> - <title><![CDATA[Guldkorn Från Mäster Cees Memoarer]]></title> - <artist><![CDATA[Cornelis Vreeswijk]]></artist> - <year>0</year> - <popularity>352</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Cornelius+Vreeswijk/Cornelis+Sjunger+Victor+Jara"> - <url>http://music.yahoo.com/Cornelius+Vreeswijk/Cornelis+Sjunger+Victor+Jara</url> - <title><![CDATA[Cornelis Sjunger Victor Jara]]></title> - <artist><![CDATA[Cornelius Vreeswijk]]></artist> - <year>0</year> - <popularity>178</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Creedence+Clearwater+Revival/Chronicle%2C+Vol.+1"> - <url>http://music.yahoo.com/Creedence+Clearwater+Revival/Chronicle%2C+Vol.+1</url> - <title><![CDATA[Chronicle, Vol. 1]]></title> - <artist><![CDATA[Creedence Clearwater Revival]]></artist> - <year>0</year> - <popularity>383</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Deep+Forest/Boheme"> - <url>http://music.yahoo.com/Deep+Forest/Boheme</url> - <title><![CDATA[Boheme]]></title> - <artist><![CDATA[Deep Forest]]></artist> - <year>0</year> - <popularity>139</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Deep+Purple/Scandinavian+Nights"> - <url>http://music.yahoo.com/Deep+Purple/Scandinavian+Nights</url> - <title><![CDATA[Scandinavian Nights]]></title> - <artist><![CDATA[Deep Purple]]></artist> - <year>0</year> - <popularity>273</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dire+Straits/Alchemy"> - <url>http://music.yahoo.com/Dire+Straits/Alchemy</url> - <title><![CDATA[Alchemy]]></title> - <artist><![CDATA[Dire Straits]]></artist> - <year>0</year> - <popularity>298</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dum+Dum+Boys/1001+Watt"> - <url>http://music.yahoo.com/Dum+Dum+Boys/1001+Watt</url> - <title><![CDATA[1001 Watt]]></title> - <artist><![CDATA[Dum Dum Boys]]></artist> - <year>0</year> - <popularity>382</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dum+Dum+Boys/Blodig+Alvor+Na+Na+Na+Na+Na"> - <url>http://music.yahoo.com/Dum+Dum+Boys/Blodig+Alvor+Na+Na+Na+Na+Na</url> - <title><![CDATA[Blodig Alvor Na Na Na Na Na]]></title> - <artist><![CDATA[Dum Dum Boys]]></artist> - <year>0</year> - <popularity>307</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dum+Dum+Boys/Ludium"> - <url>http://music.yahoo.com/Dum+Dum+Boys/Ludium</url> - <title><![CDATA[Ludium]]></title> - <artist><![CDATA[Dum Dum Boys]]></artist> - <year>0</year> - <popularity>135</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dum+Dum+Boys/Transit"> - <url>http://music.yahoo.com/Dum+Dum+Boys/Transit</url> - <title><![CDATA[Transit]]></title> - <artist><![CDATA[Dum Dum Boys]]></artist> - <year>0</year> - <popularity>207</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dumdum+Boys/0+Watt"> - <url>http://music.yahoo.com/Dumdum+Boys/0+Watt</url> - <title><![CDATA[0 Watt]]></title> - <artist><![CDATA[Dumdum Boys]]></artist> - <year>0</year> - <popularity>76</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Dumdum+Boys/Splitter+Pine"> - <url>http://music.yahoo.com/Dumdum+Boys/Splitter+Pine</url> - <title><![CDATA[Splitter Pine]]></title> - <artist><![CDATA[Dumdum Boys]]></artist> - <year>0</year> - <popularity>135</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Enya/Shepherd+Moons"> - <url>http://music.yahoo.com/Enya/Shepherd+Moons</url> - <title><![CDATA[Shepherd Moons]]></title> - <artist><![CDATA[Enya]]></artist> - <year>0</year> - <popularity>469</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Enya/Watermark"> - <url>http://music.yahoo.com/Enya/Watermark</url> - <title><![CDATA[Watermark]]></title> - <artist><![CDATA[Enya]]></artist> - <year>0</year> - <popularity>167</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eric+Clapton/Me+and+Mr.+Johnson"> - <url>http://music.yahoo.com/Eric+Clapton/Me+and+Mr.+Johnson</url> - <title><![CDATA[Me and Mr. Johnson]]></title> - <artist><![CDATA[Eric Clapton]]></artist> - <year>0</year> - <popularity>183</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Eric+Clapton+%26+B.B.+King/Riding+With+The+King"> - <url>http://music.yahoo.com/Eric+Clapton+%26+B.B.+King/Riding+With+The+King</url> - <title><![CDATA[Riding With The King]]></title> - <artist><![CDATA[Eric Clapton & B.B. King]]></artist> - <year>0</year> - <popularity>483</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ferenc+Santa+and+His+Gypsy+Band/Csa%CC%81rda%CC%81s_+Hungarian+Gypsy+Music"> - <url>http://music.yahoo.com/Ferenc+Santa+and+His+Gypsy+Band/Csa%CC%81rda%CC%81s_+Hungarian+Gypsy+Music</url> - <title><![CDATA[Csárdás_ Hungarian Gypsy Music]]></title> - <artist><![CDATA[Ferenc Santa and His Gypsy Band]]></artist> - <year>0</year> - <popularity>62</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ferenc+Santa+Jr/Alma+Cigana+-+Hungarian+Gipsy+Music"> - <url>http://music.yahoo.com/Ferenc+Santa+Jr/Alma+Cigana+-+Hungarian+Gipsy+Music</url> - <title><![CDATA[Alma Cigana - Hungarian Gipsy Music]]></title> - <artist><![CDATA[Ferenc Santa Jr]]></artist> - <year>0</year> - <popularity>166</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Fred+McDowell/The+Best+of+Mississippi"> - <url>http://music.yahoo.com/Fred+McDowell/The+Best+of+Mississippi</url> - <title><![CDATA[The Best of Mississippi]]></title> - <artist><![CDATA[Fred McDowell]]></artist> - <year>0</year> - <popularity>429</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Fun+House/Plugged%21"> - <url>http://music.yahoo.com/Fun+House/Plugged%21</url> - <title><![CDATA[Plugged!]]></title> - <artist><![CDATA[Fun House]]></artist> - <year>0</year> - <popularity>5</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Genesis/Turn+It+On+Again+-+The+Hits"> - <url>http://music.yahoo.com/Genesis/Turn+It+On+Again+-+The+Hits</url> - <title><![CDATA[Turn It On Again - The Hits]]></title> - <artist><![CDATA[Genesis]]></artist> - <year>0</year> - <popularity>123</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gipsy+Kings/Gipsy+Kings"> - <url>http://music.yahoo.com/Gipsy+Kings/Gipsy+Kings</url> - <title><![CDATA[Gipsy Kings]]></title> - <artist><![CDATA[Gipsy Kings]]></artist> - <year>0</year> - <popularity>311</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gipsy+Kings/Love+%26+Liberte%CC%81"> - <url>http://music.yahoo.com/Gipsy+Kings/Love+%26+Liberte%CC%81</url> - <title><![CDATA[Love & Liberté]]></title> - <artist><![CDATA[Gipsy Kings]]></artist> - <year>0</year> - <popularity>374</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gipsy+Kings/Mosaique"> - <url>http://music.yahoo.com/Gipsy+Kings/Mosaique</url> - <title><![CDATA[Mosaique]]></title> - <artist><![CDATA[Gipsy Kings]]></artist> - <year>0</year> - <popularity>440</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gipsy+Kings/Somos+Gitanos"> - <url>http://music.yahoo.com/Gipsy+Kings/Somos+Gitanos</url> - <title><![CDATA[Somos Gitanos]]></title> - <artist><![CDATA[Gipsy Kings]]></artist> - <year>0</year> - <popularity>454</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Gyllene+Tider/Moderna+Tider"> - <url>http://music.yahoo.com/Gyllene+Tider/Moderna+Tider</url> - <title><![CDATA[Moderna Tider]]></title> - <artist><![CDATA[Gyllene Tider]]></artist> - <year>0</year> - <popularity>486</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Henning+Sommerro+_+Dalakopa+_+Guest+Artists/Vindens+Hjul+-+Gammeldansmesse"> - <url>http://music.yahoo.com/Henning+Sommerro+_+Dalakopa+_+Guest+Artists/Vindens+Hjul+-+Gammeldansmesse</url> - <title><![CDATA[Vindens Hjul - Gammeldansmesse]]></title> - <artist><![CDATA[Henning Sommerro _ Dalakopa _ Guest Artists]]></artist> - <year>0</year> - <popularity>31</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Herbie+Hancock/Future+Shock"> - <url>http://music.yahoo.com/Herbie+Hancock/Future+Shock</url> - <title><![CDATA[Future Shock]]></title> - <artist><![CDATA[Herbie Hancock]]></artist> - <year>0</year> - <popularity>29</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ibrahim+Ferrer/Buena+Vista+Social+Club+Presents+Ibrahim+Ferrer"> - <url>http://music.yahoo.com/Ibrahim+Ferrer/Buena+Vista+Social+Club+Presents+Ibrahim+Ferrer</url> - <title><![CDATA[Buena Vista Social Club Presents Ibrahim Ferrer]]></title> - <artist><![CDATA[Ibrahim Ferrer]]></artist> - <year>0</year> - <popularity>64</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/J.+Geils+Band/Love+Stinks"> - <url>http://music.yahoo.com/J.+Geils+Band/Love+Stinks</url> - <title><![CDATA[Love Stinks]]></title> - <artist><![CDATA[J. Geils Band]]></artist> - <year>0</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/James+Heatherington+Band_Avalon+Music/Big+Band+Mambo"> - <url>http://music.yahoo.com/James+Heatherington+Band_Avalon+Music/Big+Band+Mambo</url> - <title><![CDATA[Big Band Mambo]]></title> - <artist><![CDATA[James Heatherington Band_Avalon Music]]></artist> - <year>0</year> - <popularity>10</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/James+Horner/Titanic"> - <url>http://music.yahoo.com/James+Horner/Titanic</url> - <title><![CDATA[Titanic]]></title> - <artist><![CDATA[James Horner]]></artist> - <year>0</year> - <popularity>164</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/James+Horner+-+%28Celine+Dion-Vocals%29/Titanic"> - <url>http://music.yahoo.com/James+Horner+-+%28Celine+Dion-Vocals%29/Titanic</url> - <title><![CDATA[Titanic]]></title> - <artist><![CDATA[James Horner - (Celine Dion-Vocals)]]></artist> - <year>0</year> - <popularity>30</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jan+Garbarek/Officium"> - <url>http://music.yahoo.com/Jan+Garbarek/Officium</url> - <title><![CDATA[Officium]]></title> - <artist><![CDATA[Jan Garbarek]]></artist> - <year>0</year> - <popularity>126</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jan+Garbarek+%26+The+Hilliard+Ensemble/Officium"> - <url>http://music.yahoo.com/Jan+Garbarek+%26+The+Hilliard+Ensemble/Officium</url> - <title><![CDATA[Officium]]></title> - <artist><![CDATA[Jan Garbarek & The Hilliard Ensemble]]></artist> - <year>0</year> - <popularity>67</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jan+Garbarek+%26+Ustad+Fateh+Ali+Khan/Ragas+and+Sagas"> - <url>http://music.yahoo.com/Jan+Garbarek+%26+Ustad+Fateh+Ali+Khan/Ragas+and+Sagas</url> - <title><![CDATA[Ragas and Sagas]]></title> - <artist><![CDATA[Jan Garbarek & Ustad Fateh Ali Khan]]></artist> - <year>0</year> - <popularity>246</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jan+Johansson/Folkvisor"> - <url>http://music.yahoo.com/Jan+Johansson/Folkvisor</url> - <title><![CDATA[Folkvisor]]></title> - <artist><![CDATA[Jan Johansson]]></artist> - <year>0</year> - <popularity>380</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jean-Michel+Jarre/Chronologie"> - <url>http://music.yahoo.com/Jean-Michel+Jarre/Chronologie</url> - <title><![CDATA[Chronologie]]></title> - <artist><![CDATA[Jean-Michel Jarre]]></artist> - <year>0</year> - <popularity>240</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jean-Michel+Jarre/En+Concert+Houston+-+Lyon"> - <url>http://music.yahoo.com/Jean-Michel+Jarre/En+Concert+Houston+-+Lyon</url> - <title><![CDATA[En Concert Houston - Lyon]]></title> - <artist><![CDATA[Jean-Michel Jarre]]></artist> - <year>0</year> - <popularity>464</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jean-Michel+Jarre/Hong+Kong"> - <url>http://music.yahoo.com/Jean-Michel+Jarre/Hong+Kong</url> - <title><![CDATA[Hong Kong]]></title> - <artist><![CDATA[Jean-Michel Jarre]]></artist> - <year>0</year> - <popularity>412</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jean-Michel+Jarre/Oxygene"> - <url>http://music.yahoo.com/Jean-Michel+Jarre/Oxygene</url> - <title><![CDATA[Oxygene]]></title> - <artist><![CDATA[Jean-Michel Jarre]]></artist> - <year>0</year> - <popularity>279</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Jonas+Fjeld/Nerven+I+Min+Sang"> - <url>http://music.yahoo.com/Jonas+Fjeld/Nerven+I+Min+Sang</url> - <title><![CDATA[Nerven I Min Sang]]></title> - <artist><![CDATA[Jonas Fjeld]]></artist> - <year>0</year> - <popularity>131</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes/11+ubesvarte+anrop"> - <url>http://music.yahoo.com/Kari+Bremnes/11+ubesvarte+anrop</url> - <title><![CDATA[11 ubesvarte anrop]]></title> - <artist><![CDATA[Kari Bremnes]]></artist> - <year>0</year> - <popularity>101</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes/Erindring"> - <url>http://music.yahoo.com/Kari+Bremnes/Erindring</url> - <title><![CDATA[Erindring]]></title> - <artist><![CDATA[Kari Bremnes]]></artist> - <year>0</year> - <popularity>80</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes/Loesrivese"> - <url>http://music.yahoo.com/Kari+Bremnes/Loesrivese</url> - <title><![CDATA[Loesrivese]]></title> - <artist><![CDATA[Kari Bremnes]]></artist> - <year>0</year> - <popularity>31</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes/Mitt+ville+hjerte"> - <url>http://music.yahoo.com/Kari+Bremnes/Mitt+ville+hjerte</url> - <title><![CDATA[Mitt ville hjerte]]></title> - <artist><![CDATA[Kari Bremnes]]></artist> - <year>0</year> - <popularity>389</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes/Spor"> - <url>http://music.yahoo.com/Kari+Bremnes/Spor</url> - <title><![CDATA[Spor]]></title> - <artist><![CDATA[Kari Bremnes]]></artist> - <year>0</year> - <popularity>304</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Kari+Bremnes+and+Rikard+Wolff/Desemberbarn"> - <url>http://music.yahoo.com/Kari+Bremnes+and+Rikard+Wolff/Desemberbarn</url> - <title><![CDATA[Desemberbarn]]></title> - <artist><![CDATA[Kari Bremnes and Rikard Wolff]]></artist> - <year>0</year> - <popularity>236</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Knut+Reiersrud/Soul+Of+A+Man"> - <url>http://music.yahoo.com/Knut+Reiersrud/Soul+Of+A+Man</url> - <title><![CDATA[Soul Of A Man]]></title> - <artist><![CDATA[Knut Reiersrud]]></artist> - <year>0</year> - <popularity>165</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Leadbelly/The+Definitive+Leadbelly"> - <url>http://music.yahoo.com/Leadbelly/The+Definitive+Leadbelly</url> - <title><![CDATA[The Definitive Leadbelly]]></title> - <artist><![CDATA[Leadbelly]]></artist> - <year>0</year> - <popularity>385</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Leningrad+Cowboys/Go+America"> - <url>http://music.yahoo.com/Leningrad+Cowboys/Go+America</url> - <title><![CDATA[Go America]]></title> - <artist><![CDATA[Leningrad Cowboys]]></artist> - <year>0</year> - <popularity>468</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Leningrad+Cowboys/Live+In+Prowinzz"> - <url>http://music.yahoo.com/Leningrad+Cowboys/Live+In+Prowinzz</url> - <title><![CDATA[Live In Prowinzz]]></title> - <artist><![CDATA[Leningrad Cowboys]]></artist> - <year>0</year> - <popularity>82</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Leonard+Cohen/Democracy+Is+Coming+To+The+USA"> - <url>http://music.yahoo.com/Leonard+Cohen/Democracy+Is+Coming+To+The+USA</url> - <title><![CDATA[Democracy Is Coming To The USA]]></title> - <artist><![CDATA[Leonard Cohen]]></artist> - <year>0</year> - <popularity>208</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lightnin%27+Hopkins/28+Great+Songs"> - <url>http://music.yahoo.com/Lightnin%27+Hopkins/28+Great+Songs</url> - <title><![CDATA[28 Great Songs]]></title> - <artist><![CDATA[Lightnin' Hopkins]]></artist> - <year>0</year> - <popularity>416</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lightnin%27+Hopkins/Lightnin%27+Hopkins"> - <url>http://music.yahoo.com/Lightnin%27+Hopkins/Lightnin%27+Hopkins</url> - <title><![CDATA[Lightnin' Hopkins]]></title> - <artist><![CDATA[Lightnin' Hopkins]]></artist> - <year>0</year> - <popularity>234</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Lindfors%2C+Lill%2C+o.a_/Salomos+H%C3%B8ysang"> - <url>http://music.yahoo.com/Lindfors%2C+Lill%2C+o.a_/Salomos+H%C3%B8ysang</url> - <title><![CDATA[Salomos Høysang]]></title> - <artist><![CDATA[Lindfors, Lill, o.a_]]></artist> - <year>0</year> - <popularity>204</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madonna/Ray+Of+Light"> - <url>http://music.yahoo.com/Madonna/Ray+Of+Light</url> - <title><![CDATA[Ray Of Light]]></title> - <artist><![CDATA[Madonna]]></artist> - <year>0</year> - <popularity>59</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madrugada/Industrial+Silence"> - <url>http://music.yahoo.com/Madrugada/Industrial+Silence</url> - <title><![CDATA[Industrial Silence]]></title> - <artist><![CDATA[Madrugada]]></artist> - <year>0</year> - <popularity>467</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madrugada/The+Nightly+Disease"> - <url>http://music.yahoo.com/Madrugada/The+Nightly+Disease</url> - <title><![CDATA[The Nightly Disease]]></title> - <artist><![CDATA[Madrugada]]></artist> - <year>0</year> - <popularity>69</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Madrugada/The+Nightly+Disease+Vol+II"> - <url>http://music.yahoo.com/Madrugada/The+Nightly+Disease+Vol+II</url> - <title><![CDATA[The Nightly Disease Vol II]]></title> - <artist><![CDATA[Madrugada]]></artist> - <year>0</year> - <popularity>51</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Mari+Boine/Goaskinviellja+%28_rnebror+_+Brother+Eagle%29"> - <url>http://music.yahoo.com/Mari+Boine/Goaskinviellja+%28_rnebror+_+Brother+Eagle%29</url> - <title><![CDATA[Goaskinviellja (_rnebror _ Brother Eagle)]]></title> - <artist><![CDATA[Mari Boine]]></artist> - <year>0</year> - <popularity>121</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marius+Mu%CC%88ller%27s+Funhouse/Funhouse"> - <url>http://music.yahoo.com/Marius+Mu%CC%88ller%27s+Funhouse/Funhouse</url> - <title><![CDATA[Funhouse]]></title> - <artist><![CDATA[Marius Müller's Funhouse]]></artist> - <year>0</year> - <popularity>88</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Marius+Mu%CC%88ller%27s+Funhouse/Maximum"> - <url>http://music.yahoo.com/Marius+Mu%CC%88ller%27s+Funhouse/Maximum</url> - <title><![CDATA[Maximum]]></title> - <artist><![CDATA[Marius Müller's Funhouse]]></artist> - <year>0</year> - <popularity>150</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Martinho+da+Vila/Definitivo"> - <url>http://music.yahoo.com/Martinho+da+Vila/Definitivo</url> - <title><![CDATA[Definitivo]]></title> - <artist><![CDATA[Martinho da Vila]]></artist> - <year>0</year> - <popularity>430</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Michael+_Blind-Dog_+Gatewood/I+Hear+the+Blues+Callin%27+and+the+Devil+Is+Near+%21"> - <url>http://music.yahoo.com/Michael+_Blind-Dog_+Gatewood/I+Hear+the+Blues+Callin%27+and+the+Devil+Is+Near+%21</url> - <title><![CDATA[I Hear the Blues Callin' and the Devil Is Near !]]></title> - <artist><![CDATA[Michael _Blind-Dog_ Gatewood]]></artist> - <year>0</year> - <popularity>177</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Motorpsycho/Let+Them+Eat+Cake"> - <url>http://music.yahoo.com/Motorpsycho/Let+Them+Eat+Cake</url> - <title><![CDATA[Let Them Eat Cake]]></title> - <artist><![CDATA[Motorpsycho]]></artist> - <year>0</year> - <popularity>250</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Muddy+Waters/Got+My+Mojo+Working"> - <url>http://music.yahoo.com/Muddy+Waters/Got+My+Mojo+Working</url> - <title><![CDATA[Got My Mojo Working]]></title> - <artist><![CDATA[Muddy Waters]]></artist> - <year>0</year> - <popularity>194</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nacho+Cano/Un+Mundo+Separado+Por+El+Mismo+Dios"> - <url>http://music.yahoo.com/Nacho+Cano/Un+Mundo+Separado+Por+El+Mismo+Dios</url> - <title><![CDATA[Un Mundo Separado Por El Mismo Dios]]></title> - <artist><![CDATA[Nacho Cano]]></artist> - <year>0</year> - <popularity>450</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Neil+Young/Harvest"> - <url>http://music.yahoo.com/Neil+Young/Harvest</url> - <title><![CDATA[Harvest]]></title> - <artist><![CDATA[Neil Young]]></artist> - <year>0</year> - <popularity>226</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Neil+Young/Neil+Young+Unplugged"> - <url>http://music.yahoo.com/Neil+Young/Neil+Young+Unplugged</url> - <title><![CDATA[Neil Young Unplugged]]></title> - <artist><![CDATA[Neil Young]]></artist> - <year>0</year> - <popularity>480</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Neil+Young+%26+Crazy+Horse/Sleeps+With+Angels"> - <url>http://music.yahoo.com/Neil+Young+%26+Crazy+Horse/Sleeps+With+Angels</url> - <title><![CDATA[Sleeps With Angels]]></title> - <artist><![CDATA[Neil Young & Crazy Horse]]></artist> - <year>0</year> - <popularity>255</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Netinho/Netinho+Ao+Vivo%21"> - <url>http://music.yahoo.com/Netinho/Netinho+Ao+Vivo%21</url> - <title><![CDATA[Netinho Ao Vivo!]]></title> - <artist><![CDATA[Netinho]]></artist> - <year>0</year> - <popularity>63</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Nirvana/Nevermind"> - <url>http://music.yahoo.com/Nirvana/Nevermind</url> - <title><![CDATA[Nevermind]]></title> - <artist><![CDATA[Nirvana]]></artist> - <year>0</year> - <popularity>317</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Odd+B%C3%B8rretzen%2C+Lars+Martin+Myhre/Noen+ganger+er+det+all+right"> - <url>http://music.yahoo.com/Odd+B%C3%B8rretzen%2C+Lars+Martin+Myhre/Noen+ganger+er+det+all+right</url> - <title><![CDATA[Noen ganger er det all right]]></title> - <artist><![CDATA[Odd Børretzen, Lars Martin Myhre]]></artist> - <year>0</year> - <popularity>56</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Omara+Portuondo/Buena+Vista+Social+Club+Presents+Omara+Portuondo"> - <url>http://music.yahoo.com/Omara+Portuondo/Buena+Vista+Social+Club+Presents+Omara+Portuondo</url> - <title><![CDATA[Buena Vista Social Club Presents Omara Portuondo]]></title> - <artist><![CDATA[Omara Portuondo]]></artist> - <year>0</year> - <popularity>287</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Omara+Portuondo/La+Coleccio%CC%81n+Cubana"> - <url>http://music.yahoo.com/Omara+Portuondo/La+Coleccio%CC%81n+Cubana</url> - <title><![CDATA[La Colección Cubana]]></title> - <artist><![CDATA[Omara Portuondo]]></artist> - <year>0</year> - <popularity>79</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/A+Momentary+Lapse+Of+Reason"> - <url>http://music.yahoo.com/Pink+Floyd/A+Momentary+Lapse+Of+Reason</url> - <title><![CDATA[A Momentary Lapse Of Reason]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>0</year> - <popularity>391</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/The+Wall"> - <url>http://music.yahoo.com/Pink+Floyd/The+Wall</url> - <title><![CDATA[The Wall]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>0</year> - <popularity>298</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Pink+Floyd/Wish+You+Were+Here"> - <url>http://music.yahoo.com/Pink+Floyd/Wish+You+Were+Here</url> - <title><![CDATA[Wish You Were Here]]></title> - <artist><![CDATA[Pink Floyd]]></artist> - <year>1975</year> - <popularity>481</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Povl+Dissing%2C+Knut+Reiersrud%2C+Iver+Kleive/Den+Signede+Dag"> - <url>http://music.yahoo.com/Povl+Dissing%2C+Knut+Reiersrud%2C+Iver+Kleive/Den+Signede+Dag</url> - <title><![CDATA[Den Signede Dag]]></title> - <artist><![CDATA[Povl Dissing, Knut Reiersrud, Iver Kleive]]></artist> - <year>0</year> - <popularity>414</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Quincy+Jones/Big+Band+Bossa+Nova"> - <url>http://music.yahoo.com/Quincy+Jones/Big+Band+Bossa+Nova</url> - <title><![CDATA[Big Band Bossa Nova]]></title> - <artist><![CDATA[Quincy Jones]]></artist> - <year>0</year> - <popularity>213</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/R.E.M_/Pop+Song"> - <url>http://music.yahoo.com/R.E.M_/Pop+Song</url> - <title><![CDATA[Pop Song]]></title> - <artist><![CDATA[R.E.M_]]></artist> - <year>0</year> - <popularity>384</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/Blaff"> - <url>http://music.yahoo.com/Raga+Rockers/Blaff</url> - <title><![CDATA[Blaff]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>0</year> - <popularity>19</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/Forbudte+F%C3%B8lelser"> - <url>http://music.yahoo.com/Raga+Rockers/Forbudte+F%C3%B8lelser</url> - <title><![CDATA[Forbudte Følelser]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>0</year> - <popularity>268</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Raga+Rockers/Rock%27N%27Roll+Party"> - <url>http://music.yahoo.com/Raga+Rockers/Rock%27N%27Roll+Party</url> - <title><![CDATA[Rock'N'Roll Party]]></title> - <artist><![CDATA[Raga Rockers]]></artist> - <year>0</year> - <popularity>276</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rickie+Lee+Jones/Pop+Pop"> - <url>http://music.yahoo.com/Rickie+Lee+Jones/Pop+Pop</url> - <title><![CDATA[Pop Pop]]></title> - <artist><![CDATA[Rickie Lee Jones]]></artist> - <year>0</year> - <popularity>189</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ricky+Martin/La+Copa+De+La+Vida+%28Single%29"> - <url>http://music.yahoo.com/Ricky+Martin/La+Copa+De+La+Vida+%28Single%29</url> - <title><![CDATA[La Copa De La Vida (Single)]]></title> - <artist><![CDATA[Ricky Martin]]></artist> - <year>0</year> - <popularity>180</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Roger+Waters/What+God+Wants%2C+Part+I+%285_+EP%29"> - <url>http://music.yahoo.com/Roger+Waters/What+God+Wants%2C+Part+I+%285_+EP%29</url> - <title><![CDATA[What God Wants, Part I (5_ EP)]]></title> - <artist><![CDATA[Roger Waters]]></artist> - <year>0</year> - <popularity>56</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rosana/Lunas+Rotas"> - <url>http://music.yahoo.com/Rosana/Lunas+Rotas</url> - <title><![CDATA[Lunas Rotas]]></title> - <artist><![CDATA[Rosana]]></artist> - <year>0</year> - <popularity>282</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rube%CC%81n+Gonza%CC%81lez/Chanchullo"> - <url>http://music.yahoo.com/Rube%CC%81n+Gonza%CC%81lez/Chanchullo</url> - <title><![CDATA[Chanchullo]]></title> - <artist><![CDATA[Rubén González]]></artist> - <year>0</year> - <popularity>113</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Rube%CC%81n+Gonza%CC%81lez/Introducing...Ruben+Gonzalez"> - <url>http://music.yahoo.com/Rube%CC%81n+Gonza%CC%81lez/Introducing...Ruben+Gonzalez</url> - <title><![CDATA[Introducing...Ruben Gonzalez]]></title> - <artist><![CDATA[Rubén González]]></artist> - <year>0</year> - <popularity>260</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Ry+Cooder/Paris%2C+Texas"> - <url>http://music.yahoo.com/Ry+Cooder/Paris%2C+Texas</url> - <title><![CDATA[Paris, Texas]]></title> - <artist><![CDATA[Ry Cooder]]></artist> - <year>0</year> - <popularity>451</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigmund+Groven/Natt%C3%B8nsker"> - <url>http://music.yahoo.com/Sigmund+Groven/Natt%C3%B8nsker</url> - <title><![CDATA[Nattønsker]]></title> - <artist><![CDATA[Sigmund Groven]]></artist> - <year>0</year> - <popularity>80</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sigmund+Groven_Arve+Tellefsen/Musikken+inni+oss"> - <url>http://music.yahoo.com/Sigmund+Groven_Arve+Tellefsen/Musikken+inni+oss</url> - <title><![CDATA[Musikken inni oss]]></title> - <artist><![CDATA[Sigmund Groven_Arve Tellefsen]]></artist> - <year>0</year> - <popularity>262</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Simon+%26+Garfunkel/The+Definitive+Simon+%26+Garfunkel"> - <url>http://music.yahoo.com/Simon+%26+Garfunkel/The+Definitive+Simon+%26+Garfunkel</url> - <title><![CDATA[The Definitive Simon & Garfunkel]]></title> - <artist><![CDATA[Simon & Garfunkel]]></artist> - <year>0</year> - <popularity>304</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Simple+Minds/Live+in+the+City+of+Light"> - <url>http://music.yahoo.com/Simple+Minds/Live+in+the+City+of+Light</url> - <title><![CDATA[Live in the City of Light]]></title> - <artist><![CDATA[Simple Minds]]></artist> - <year>0</year> - <popularity>44</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Simple+Minds/Street+Fighting+Years"> - <url>http://music.yahoo.com/Simple+Minds/Street+Fighting+Years</url> - <title><![CDATA[Street Fighting Years]]></title> - <artist><![CDATA[Simple Minds]]></artist> - <year>0</year> - <popularity>69</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Son+House/Heroes+Of+The+Blues_+Very+Best+Of+Son+House"> - <url>http://music.yahoo.com/Son+House/Heroes+Of+The+Blues_+Very+Best+Of+Son+House</url> - <title><![CDATA[Heroes Of The Blues_ Very Best Of Son House]]></title> - <artist><![CDATA[Son House]]></artist> - <year>0</year> - <popularity>417</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Stan+Getz+%26+Joa%CC%83o+Gilberto/Getz_Gilberto"> - <url>http://music.yahoo.com/Stan+Getz+%26+Joa%CC%83o+Gilberto/Getz_Gilberto</url> - <title><![CDATA[Getz_Gilberto]]></title> - <artist><![CDATA[Stan Getz & João Gilberto]]></artist> - <year>0</year> - <popularity>312</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sting/_...Nada+Como+El+Sol"> - <url>http://music.yahoo.com/Sting/_...Nada+Como+El+Sol</url> - <title><![CDATA[_...Nada Como El Sol]]></title> - <artist><![CDATA[Sting]]></artist> - <year>0</year> - <popularity>66</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sting/Brand+New+Day"> - <url>http://music.yahoo.com/Sting/Brand+New+Day</url> - <title><![CDATA[Brand New Day]]></title> - <artist><![CDATA[Sting]]></artist> - <year>0</year> - <popularity>117</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Sting/The+Very+Best+Of+Sting+%26+The+Police"> - <url>http://music.yahoo.com/Sting/The+Very+Best+Of+Sting+%26+The+Police</url> - <title><![CDATA[The Very Best Of Sting & The Police]]></title> - <artist><![CDATA[Sting]]></artist> - <year>0</year> - <popularity>8</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/T.Rex/Electric+Warrior"> - <url>http://music.yahoo.com/T.Rex/Electric+Warrior</url> - <title><![CDATA[Electric Warrior]]></title> - <artist><![CDATA[T.Rex]]></artist> - <year>0</year> - <popularity>55</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Fear+Of+Music"> - <url>http://music.yahoo.com/Talking+Heads/Fear+Of+Music</url> - <title><![CDATA[Fear Of Music]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>457</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Little+Creatures"> - <url>http://music.yahoo.com/Talking+Heads/Little+Creatures</url> - <title><![CDATA[Little Creatures]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>382</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/More+Songs+About+Buildings+And+Food"> - <url>http://music.yahoo.com/Talking+Heads/More+Songs+About+Buildings+And+Food</url> - <title><![CDATA[More Songs About Buildings And Food]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>93</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Naked"> - <url>http://music.yahoo.com/Talking+Heads/Naked</url> - <title><![CDATA[Naked]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>363</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Speaking+in+Tongues"> - <url>http://music.yahoo.com/Talking+Heads/Speaking+in+Tongues</url> - <title><![CDATA[Speaking in Tongues]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>65</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Stop+Making+Sense"> - <url>http://music.yahoo.com/Talking+Heads/Stop+Making+Sense</url> - <title><![CDATA[Stop Making Sense]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>255</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/Talking+Heads_+77"> - <url>http://music.yahoo.com/Talking+Heads/Talking+Heads_+77</url> - <title><![CDATA[Talking Heads_ 77]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>228</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Talking+Heads/True+Stories"> - <url>http://music.yahoo.com/Talking+Heads/True+Stories</url> - <title><![CDATA[True Stories]]></title> - <artist><![CDATA[Talking Heads]]></artist> - <year>0</year> - <popularity>361</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tango+Concertino/M_+Marthe+Werring"> - <url>http://music.yahoo.com/Tango+Concertino/M_+Marthe+Werring</url> - <title><![CDATA[M_ Marthe Werring]]></title> - <artist><![CDATA[Tango Concertino]]></artist> - <year>0</year> - <popularity>328</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Bravado/Harmonica+Man"> - <url>http://music.yahoo.com/The+Bravado/Harmonica+Man</url> - <title><![CDATA[Harmonica Man]]></title> - <artist><![CDATA[The Bravado]]></artist> - <year>0</year> - <popularity>24</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Cure/Kiss+Me%2C+Kiss+Me%2C+Kiss+Me"> - <url>http://music.yahoo.com/The+Cure/Kiss+Me%2C+Kiss+Me%2C+Kiss+Me</url> - <title><![CDATA[Kiss Me, Kiss Me, Kiss Me]]></title> - <artist><![CDATA[The Cure]]></artist> - <year>0</year> - <popularity>336</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/Alive+She+Cried+%5BLive%5D"> - <url>http://music.yahoo.com/The+Doors/Alive+She+Cried+%5BLive%5D</url> - <title><![CDATA[Alive She Cried [Live]]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>210</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/Morrison+Hotel"> - <url>http://music.yahoo.com/The+Doors/Morrison+Hotel</url> - <title><![CDATA[Morrison Hotel]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>365</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/Strange+Days"> - <url>http://music.yahoo.com/The+Doors/Strange+Days</url> - <title><![CDATA[Strange Days]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>163</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/The+Doors"> - <url>http://music.yahoo.com/The+Doors/The+Doors</url> - <title><![CDATA[The Doors]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>181</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/The+Soft+Parade"> - <url>http://music.yahoo.com/The+Doors/The+Soft+Parade</url> - <title><![CDATA[The Soft Parade]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>395</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Doors/Waiting+For+The+Sun"> - <url>http://music.yahoo.com/The+Doors/Waiting+For+The+Sun</url> - <title><![CDATA[Waiting For The Sun]]></title> - <artist><![CDATA[The Doors]]></artist> - <year>0</year> - <popularity>19</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Pogues/If+I+Should+Fall+From+Grace+With+God"> - <url>http://music.yahoo.com/The+Pogues/If+I+Should+Fall+From+Grace+With+God</url> - <title><![CDATA[If I Should Fall From Grace With God]]></title> - <artist><![CDATA[The Pogues]]></artist> - <year>0</year> - <popularity>88</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Pogues/Peace+And+Love"> - <url>http://music.yahoo.com/The+Pogues/Peace+And+Love</url> - <title><![CDATA[Peace And Love]]></title> - <artist><![CDATA[The Pogues]]></artist> - <year>0</year> - <popularity>155</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Police/The+Very+Best+Of+Sting+%26+The+Police"> - <url>http://music.yahoo.com/The+Police/The+Very+Best+Of+Sting+%26+The+Police</url> - <title><![CDATA[The Very Best Of Sting & The Police]]></title> - <artist><![CDATA[The Police]]></artist> - <year>0</year> - <popularity>301</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Rainmakers/Flirting+With+The+Universe"> - <url>http://music.yahoo.com/The+Rainmakers/Flirting+With+The+Universe</url> - <title><![CDATA[Flirting With The Universe]]></title> - <artist><![CDATA[The Rainmakers]]></artist> - <year>0</year> - <popularity>350</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Rainmakers/Oslo+-+Wichita+Live"> - <url>http://music.yahoo.com/The+Rainmakers/Oslo+-+Wichita+Live</url> - <title><![CDATA[Oslo - Wichita Live]]></title> - <artist><![CDATA[The Rainmakers]]></artist> - <year>0</year> - <popularity>273</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Rainmakers/The+Good+News+And+The+Bad+News"> - <url>http://music.yahoo.com/The+Rainmakers/The+Good+News+And+The+Bad+News</url> - <title><![CDATA[The Good News And The Bad News]]></title> - <artist><![CDATA[The Rainmakers]]></artist> - <year>0</year> - <popularity>392</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Rainmakers/The+Rainmakers"> - <url>http://music.yahoo.com/The+Rainmakers/The+Rainmakers</url> - <title><![CDATA[The Rainmakers]]></title> - <artist><![CDATA[The Rainmakers]]></artist> - <year>0</year> - <popularity>411</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Rainmakers/Tornado"> - <url>http://music.yahoo.com/The+Rainmakers/Tornado</url> - <title><![CDATA[Tornado]]></title> - <artist><![CDATA[The Rainmakers]]></artist> - <year>0</year> - <popularity>204</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Triffids/Australian+Melodrama"> - <url>http://music.yahoo.com/The+Triffids/Australian+Melodrama</url> - <title><![CDATA[Australian Melodrama]]></title> - <artist><![CDATA[The Triffids]]></artist> - <year>9</year> - <popularity>315</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Waterboys/A+Pagan+Place"> - <url>http://music.yahoo.com/The+Waterboys/A+Pagan+Place</url> - <title><![CDATA[A Pagan Place]]></title> - <artist><![CDATA[The Waterboys]]></artist> - <year>0</year> - <popularity>354</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Waterboys/Dream+Harder"> - <url>http://music.yahoo.com/The+Waterboys/Dream+Harder</url> - <title><![CDATA[Dream Harder]]></title> - <artist><![CDATA[The Waterboys]]></artist> - <year>0</year> - <popularity>304</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/The+Waterboys/This+Is+The+Sea"> - <url>http://music.yahoo.com/The+Waterboys/This+Is+The+Sea</url> - <title><![CDATA[This Is The Sea]]></title> - <artist><![CDATA[The Waterboys]]></artist> - <year>0</year> - <popularity>397</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tish+Hinojosa/Destiny%27s+Gate"> - <url>http://music.yahoo.com/Tish+Hinojosa/Destiny%27s+Gate</url> - <title><![CDATA[Destiny's Gate]]></title> - <artist><![CDATA[Tish Hinojosa]]></artist> - <year>0</year> - <popularity>153</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tish+Hinojosa/Frontejas"> - <url>http://music.yahoo.com/Tish+Hinojosa/Frontejas</url> - <title><![CDATA[Frontejas]]></title> - <artist><![CDATA[Tish Hinojosa]]></artist> - <year>0</year> - <popularity>325</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Toquinho/O+Melhor+De+Toquinho"> - <url>http://music.yahoo.com/Toquinho/O+Melhor+De+Toquinho</url> - <title><![CDATA[O Melhor De Toquinho]]></title> - <artist><![CDATA[Toquinho]]></artist> - <year>0</year> - <popularity>377</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Traveling+Wilburys/Traveling+Wilburys%2C+Vol.+1"> - <url>http://music.yahoo.com/Traveling+Wilburys/Traveling+Wilburys%2C+Vol.+1</url> - <title><![CDATA[Traveling Wilburys, Vol. 1]]></title> - <artist><![CDATA[Traveling Wilburys]]></artist> - <year>0</year> - <popularity>8</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Tre+sma%CC%8A+kinesere/Luftpalass"> - <url>http://music.yahoo.com/Tre+sma%CC%8A+kinesere/Luftpalass</url> - <title><![CDATA[Luftpalass]]></title> - <artist><![CDATA[Tre små kinesere]]></artist> - <year>0</year> - <popularity>476</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Trio+Troika/A+Good+Evening+To+You"> - <url>http://music.yahoo.com/Trio+Troika/A+Good+Evening+To+You</url> - <title><![CDATA[A Good Evening To You]]></title> - <artist><![CDATA[Trio Troika]]></artist> - <year>0</year> - <popularity>404</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Achtung+Baby"> - <url>http://music.yahoo.com/U2/Achtung+Baby</url> - <title><![CDATA[Achtung Baby]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>427</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/All+That+You+Can%27t+Leave+Behind"> - <url>http://music.yahoo.com/U2/All+That+You+Can%27t+Leave+Behind</url> - <title><![CDATA[All That You Can't Leave Behind]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>400</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/B-Sides+1980-1990"> - <url>http://music.yahoo.com/U2/B-Sides+1980-1990</url> - <title><![CDATA[B-Sides 1980-1990]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>488</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Boy"> - <url>http://music.yahoo.com/U2/Boy</url> - <title><![CDATA[Boy]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>133</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/I+Still+Haven%27t+Found+What+I%27m+Looking+For+%28Single%29"> - <url>http://music.yahoo.com/U2/I+Still+Haven%27t+Found+What+I%27m+Looking+For+%28Single%29</url> - <title><![CDATA[I Still Haven't Found What I'm Looking For (Single)]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>314</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/October"> - <url>http://music.yahoo.com/U2/October</url> - <title><![CDATA[October]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>385</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Pop"> - <url>http://music.yahoo.com/U2/Pop</url> - <title><![CDATA[Pop]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>428</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Rattle+And+Hum"> - <url>http://music.yahoo.com/U2/Rattle+And+Hum</url> - <title><![CDATA[Rattle And Hum]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>246</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/The+Best+Of+1980-1990"> - <url>http://music.yahoo.com/U2/The+Best+Of+1980-1990</url> - <title><![CDATA[The Best Of 1980-1990]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>241</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/The+Unforgettable+Fire"> - <url>http://music.yahoo.com/U2/The+Unforgettable+Fire</url> - <title><![CDATA[The Unforgettable Fire]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>275</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Under+A+Blood+Red+Sky"> - <url>http://music.yahoo.com/U2/Under+A+Blood+Red+Sky</url> - <title><![CDATA[Under A Blood Red Sky]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>352</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/War"> - <url>http://music.yahoo.com/U2/War</url> - <title><![CDATA[War]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>295</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Where+The+Streets+Have+No+Name+%5BSingle%5D"> - <url>http://music.yahoo.com/U2/Where+The+Streets+Have+No+Name+%5BSingle%5D</url> - <title><![CDATA[Where The Streets Have No Name [Single]]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>18</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/With+or+Without+You+%28Single%29"> - <url>http://music.yahoo.com/U2/With+or+Without+You+%28Single%29</url> - <title><![CDATA[With or Without You (Single)]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>362</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2/Zooropa"> - <url>http://music.yahoo.com/U2/Zooropa</url> - <title><![CDATA[Zooropa]]></title> - <artist><![CDATA[U2]]></artist> - <year>0</year> - <popularity>124</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/U2+Feat.+Johnny+Cash/Zooropa"> - <url>http://music.yahoo.com/U2+Feat.+Johnny+Cash/Zooropa</url> - <title><![CDATA[Zooropa]]></title> - <artist><![CDATA[U2 Feat. Johnny Cash]]></artist> - <year>0</year> - <popularity>462</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/V+mlingan/_rderpop"> - <url>http://music.yahoo.com/V+mlingan/_rderpop</url> - <title><![CDATA[_rderpop]]></title> - <artist><![CDATA[V mlingan]]></artist> - <year>0</year> - <popularity>6</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Vamp/Ei+med+alt"> - <url>http://music.yahoo.com/Vamp/Ei+med+alt</url> - <title><![CDATA[Ei med alt]]></title> - <artist><![CDATA[Vamp]]></artist> - <year>0</year> - <popularity>277</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Vangelis/1492_+The+Conquest+Of+Paradise"> - <url>http://music.yahoo.com/Vangelis/1492_+The+Conquest+Of+Paradise</url> - <title><![CDATA[1492_ The Conquest Of Paradise]]></title> - <artist><![CDATA[Vangelis]]></artist> - <year>0</year> - <popularity>467</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Yanni/Live+At+The+Acropolis"> - <url>http://music.yahoo.com/Yanni/Live+At+The+Acropolis</url> - <title><![CDATA[Live At The Acropolis]]></title> - <artist><![CDATA[Yanni]]></artist> - <year>0</year> - <popularity>240</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Yello/Pocket+Universe+%5BBonus+Track%5D"> - <url>http://music.yahoo.com/Yello/Pocket+Universe+%5BBonus+Track%5D</url> - <title><![CDATA[Pocket Universe [Bonus Track]]]></title> - <artist><![CDATA[Yello]]></artist> - <year>0</year> - <popularity>198</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/Zbigniew+Preisner/La+double+vie+de+Ve%CC%81ronique"> - <url>http://music.yahoo.com/Zbigniew+Preisner/La+double+vie+de+Ve%CC%81ronique</url> - <title><![CDATA[La double vie de Véronique]]></title> - <artist><![CDATA[Zbigniew Preisner]]></artist> - <year>0</year> - <popularity>467</popularity> - </document> -</vespafeed> diff --git a/vespaapplication/examples/musicdata.xml b/vespaapplication/examples/musicdata.xml deleted file mode 100644 index 24ebb8d87f3..00000000000 --- a/vespaapplication/examples/musicdata.xml +++ /dev/null @@ -1,21 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/bobdylan/BestOf"> - <url>http://music.yahoo.com/bobdylan/BestOf</url> - <title>Best of Bob Dylan</title> - <artist>Bob Dylan</artist> - <year>1997</year> - <popularity>100</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/metallica/BestOf"> - <url>http://music.yahoo.com/metallica/BestOf</url> - <title>Best of Metallica</title> - <artist>Metallica</artist> - <year>2001</year> - <popularity>200</popularity> - </document> - -</vespafeed> diff --git a/vespaapplication/examples/vds-musicdata.xml b/vespaapplication/examples/vds-musicdata.xml deleted file mode 100644 index bd5dee3776a..00000000000 --- a/vespaapplication/examples/vds-musicdata.xml +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/bobdylan/BestOf"> - <title>Best of Bob Dylan</title> - <artist>Bob Dylan</artist> - <year>1997</year> - <popularity>100</popularity> - </document> - - <document type="music" documentid="id:sampleapp:music::http://music.yahoo.com/metallica/BestOf"> - <title>Best of Metallica</title> - <artist>Metallica</artist> - <year>2001</year> - <popularity>200</popularity> - </document> - -</vespafeed> diff --git a/vespaapplication/examples/vds-simple/hosts.xml b/vespaapplication/examples/vds-simple/hosts.xml deleted file mode 100644 index 51f7359aa67..00000000000 --- a/vespaapplication/examples/vds-simple/hosts.xml +++ /dev/null @@ -1,8 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> - diff --git a/vespaapplication/examples/vds-simple/searchdefinitions/music.sd b/vespaapplication/examples/vds-simple/searchdefinitions/music.sd deleted file mode 100644 index 07f0fd6c6ee..00000000000 --- a/vespaapplication/examples/vds-simple/searchdefinitions/music.sd +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search music { - document music { - - field title type string { header } - - field artist type string { header } - - field year type int { header} - - field popularity type int { header} - - field url type uri { header } - - field body type raw { body } - } -} - diff --git a/vespaapplication/examples/vds-simple/services.xml b/vespaapplication/examples/vds-simple/services.xml deleted file mode 100644 index 879e2d3ae74..00000000000 --- a/vespaapplication/examples/vds-simple/services.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - </admin> - - <jdisc version="1.0"> - <document-api/> - - <nodes> - <node hostalias="node1" /> - </nodes> - </jdisc> - - <content version="1.0" id="search"> - <redundancy>1</redundancy> - - <engine> - <vds/> - </engine> - - <documents> - <document type="music" mode="store-only"/> - </documents> - - <nodes> - <node hostalias="node1" distribution-key="0"/> - </nodes> - </content> - -</services> diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index def88408fd5..55eef2fd8d8 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -36,4 +36,8 @@ vespa_install_script(src/vespa-start-services.sh vespa-start-services bin) vespa_install_script(src/vespa-stop-configserver.sh vespa-stop-configserver bin) vespa_install_script(src/vespa-stop-services.sh vespa-stop-services bin) +configure_file(src/vespa.service.in src/vespa.service @ONLY) +configure_file(src/vespa-configserver.service.in src/vespa-configserver.service @ONLY) +install(FILES src/vespa.service src/vespa-configserver.service DESTINATION etc/systemd/system) + install(FILES src/Defaults.pm DESTINATION lib/perl5/site_perl/Yahoo/Vespa) diff --git a/vespabase/src/.gitignore b/vespabase/src/.gitignore index a76bc1497b9..63aee122f89 100644 --- a/vespabase/src/.gitignore +++ b/vespabase/src/.gitignore @@ -1,2 +1,4 @@ build lib +/vespa-configserver.service +/vespa.service diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index cbc096dc1bf..e2e20b9bd8f 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -100,7 +100,12 @@ populate_environment () { # VESPA_USE_NO_VESPAMALLOC - list of programs that should use normal system malloc read_conf_file - consider_fallback ROOT ${VESPA_HOME} + consider_fallback ROOT ${VESPA_HOME%/} + if id yahoo >/dev/null 2>&1 ; then + consider_fallback VESPA_USER "yahoo" + elif id vespa >/dev/null 2>&1 ; then + consider_fallback VESPA_USER "vespa" + fi } populate_environment @@ -154,11 +159,11 @@ fixlimits () { } checkjava () { - if java -version 2>&1 | grep "Java HotSpot.* 64-Bit Server VM" >/dev/null ; then + if java -version 2>&1 | grep "64-Bit Server VM" >/dev/null ; then : OK else echo - echo "java must invoke the 64-bit Sun Java VM" + echo "java must invoke the 64-bit Java VM" echo "Got:" java -version echo "Path: $PATH" diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 761fd6af334..86eb96093bc 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -61,6 +61,7 @@ findroot # END environment bootstrap section [ "$VESPA_HOME" ] || { echo "Missing VESPA_HOME variable" 1>&2; exit 1; } +[ "$VESPA_USER" ] || { echo "Missing VESPA_USER variable" 1>&2; exit 1; } cd $VESPA_HOME || { echo "Cannot cd to $VESPA_HOME" 1>&2; exit 1; } @@ -77,43 +78,35 @@ fixdir () { # BEGIN directory fixups -fixdir yahoo wheel 755 libdata/yell/exception -fixdir yahoo wheel 775 libexec/vespa/modelplugins -fixdir yahoo wheel 755 libexec/vespa/plugins/qrs -fixdir root wheel 1777 logs -fixdir yahoo wheel 1777 logs/vespa -fixdir yahoo wheel 755 logs/vespa/qrs -fixdir yahoo wheel 755 logs/vespa/search -fixdir root wheel 1777 tmp -fixdir yahoo wheel 1777 tmp/vespa -fixdir yahoo wheel 755 var/cache/vespa/config -fixdir yahoo wheel 1777 var/crash -fixdir yahoo wheel 755 var/db/vespa -fixdir yahoo wheel 755 var/db/vespa/config_server -fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb -fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/applications -fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/configs -fixdir yahoo wheel 755 var/db/vespa/config_server/serverdb/configs/application -fixdir yahoo wheel 755 var/db/vespa/index -fixdir yahoo wheel 755 var/db/vespa/search -fixdir yahoo wheel 755 var/db/vespa/logcontrol -fixdir root wheel 1777 var/run - -chown -hR yahoo logs/vespa -chown -hR yahoo var/db/vespa +fixdir root wheel 1777 logs +fixdir root wheel 1777 tmp +fixdir root wheel 1777 var/run +fixdir ${VESPA_USER} wheel 1777 var/crash +fixdir ${VESPA_USER} wheel 1777 logs/vespa +fixdir ${VESPA_USER} wheel 1777 tmp/vespa +fixdir ${VESPA_USER} wheel 755 libdata/yell/exception +fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs +fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs +fixdir ${VESPA_USER} wheel 755 logs/vespa/search +fixdir ${VESPA_USER} wheel 755 var/cache/vespa/config +fixdir ${VESPA_USER} wheel 755 var/db/vespa +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/applications +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/configs +fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/configs/application +fixdir ${VESPA_USER} wheel 755 var/db/vespa/index +fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol +fixdir ${VESPA_USER} wheel 755 var/db/vespa/search +fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache +fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver +fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ +fixdir ${VESPA_USER} wheel 775 libexec/vespa/modelplugins + +chown -hR ${VESPA_USER} logs/vespa +chown -hR ${VESPA_USER} var/db/vespa # END directory fixups # Delete temporary files created by storage when running. rm -f /home/y/tmp/hostinfo.*.*.report - -# Add $VESPA_HOME/bin to default path -perl -pi -e 'm=^pathmunge /usr/X11R6/bin after= and s=^=pathmunge /home/y/bin after; =' /etc/profile - -#Enable core files by default -perl -pi -e 's/^# No core files by default/# Vespa: Enable core files by default/' /etc/profile -perl -pi -e 's/^ulimit -S -c 0/ulimit -S -c unlimited/' /etc/profile - -# Don't fail script if this command fails. -# * sysctl will always return error on openvz jails -sysctl kernel.core_pattern="|/home/y/bin/vespa-core-dumper /home/y/bin64/lz4 /home/y/var/crash/%e.core.%p.lz4" || true diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in new file mode 100644 index 00000000000..8928fe87f20 --- /dev/null +++ b/vespabase/src/vespa-configserver.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Vertical Search Platform Config Server + +[Service] +Type=forking +PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/configserver.pid +ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-configserver +ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/vespabase/src/vespa-core-dumper.sh b/vespabase/src/vespa-core-dumper.sh deleted file mode 100755 index 5f9baadf4fc..00000000000 --- a/vespabase/src/vespa-core-dumper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -log_message () { - echo "warning $*" | logger -t vespa-core-dumper -} - -compressor=$1 -corefile=$2 -option=$3 - -log_message "Starting $compressor > $corefile" - - -if [ -f "$corefile" ] -then - if [ "$option" != "overwrite" ] - then - log_message "$corefile is read-only. Core is not dumped." - exit 1 - else - log_message "Overwriting $corefile" - fi -fi - -$compressor > $corefile -chmod 444 $corefile -log_message "Finished $compressor > $corefile" diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in new file mode 100644 index 00000000000..f46c7ecdcb6 --- /dev/null +++ b/vespabase/src/vespa.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Vertical Search Platform + +[Service] +Type=forking +PIDFile=@CMAKE_INSTALL_PREFIX@/var/run/sentinel.pid +ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-services +ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/vespaclient-container-plugin/OWNERS b/vespaclient-container-plugin/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/vespaclient-container-plugin/OWNERS +++ b/vespaclient-container-plugin/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 017216c3cbd..422565f55ae 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-container-plugin</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java index 4dc47f20889..bd6d2f130ea 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java @@ -12,7 +12,7 @@ import java.nio.charset.StandardCharsets; /** * Handling data from visit. * - * @author dybdahl + * @author dybis */ class LocalDataVisitorHandler extends DumpVisitorDataHandler { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java index 240cc3c3c61..fb1b317399c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -8,7 +8,7 @@ import java.util.Optional; /** * Abstract the backend stuff for the REST API, such as retrieving or updating documents. * - * @author dybdahl + * @author dybis */ public interface OperationHandler { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 21fd930e9d5..24f9873da53 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -32,7 +32,7 @@ import java.util.Set; /** * Sends operations to messagebus via document api. * - * @author dybdahl + * @author dybis */ public class OperationHandlerImpl implements OperationHandler { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java index b553d83d848..77552a61317 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java @@ -4,7 +4,7 @@ package com.yahoo.document.restapi; /** * Exceptions for Rest API * - * @author dybdahl + * @author dybis */ public class RestApiException extends Exception { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java index 98293508168..f50e7c247b1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java @@ -15,7 +15,7 @@ import static com.yahoo.jdisc.Response.Status.*; /** * Represents the request URI with its values. * - * @author dybdahl + * @author dybis */ public class RestUri { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 8b863a69f5c..c016241c095 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * API for handling single operation on a document and visiting. * - * @author dybdahl + * @author dybis */ public class RestApi extends LoggingRequestHandler { @@ -226,4 +226,4 @@ public class RestApi extends LoggingRequestHandler { }; return httpResponse; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java index cd61ae91dae..33b24e4b5e9 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -20,7 +20,7 @@ import com.yahoo.yolean.Exceptions; * * This implementation is based on V2, but the code is restructured. * - * @author dybdahl + * @author dybis */ class DocumentOperationMessageV3 { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 3c532d94b24..2e16b4a2dd0 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -278,16 +278,12 @@ public class FeedHandler extends LoggingRequestHandler { } private static String resolveLocalHostname() { - try { - InetAddress inetAddress = LinuxInetAddress.getLocalHost(); - String hostname = inetAddress.getCanonicalHostName(); - if (hostname.equals("localhost")) { - return ""; - } - return inetAddress.getCanonicalHostName(); - } catch (UnknownHostException e) { + InetAddress inetAddress = LinuxInetAddress.getLocalHost(); + String hostname = inetAddress.getCanonicalHostName(); + if (hostname.equals("localhost")) { return ""; } + return hostname; } /** diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 4694563ecbe..50ce18d1524 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -33,7 +33,7 @@ import java.util.logging.Logger; * The new API has more logic for shutting down cleanly as the server is more likely to be upgraded. * The code is restructured a bit. * - * @author dybdahl + * @author dybis */ public class FeedHandlerV3 extends LoggingRequestHandler { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java index 67d6acd926a..23d4bc7f3b7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -11,7 +11,7 @@ import java.io.InputStream; /** * Class for creating FeedReader based on dataFormat. - * @author dybdahl + * @author dybis */ public class FeedReaderFactory { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 0453d41fab8..0728106da7c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -16,7 +16,7 @@ import java.util.zip.GZIPInputStream; /** * This code is based on v2 code, but restructured so stream reading code is in one dedicated class. - * @author dybdahl + * @author dybis */ public class StreamReaderV3 { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java index cfb120d9891..1c0252704ea 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java @@ -12,7 +12,7 @@ import java.util.concurrent.Executor; /** * For setting up RestApi with a simple document type manager. * - * @author dybdahl + * @author dybis */ public class RestApiWithTestDocumentHandler extends RestApi{ diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java index 6d0ee59c68d..2910b7ad443 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.when; /** * Unit test for FeedHandler class. * - * @author dybdahl + * @author dybis */ public class FeedHandlerTest { @@ -100,4 +100,4 @@ public class FeedHandlerTest { } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java index c245490b1d7..726ed0ebb17 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -9,7 +9,7 @@ import java.io.InputStream; /** * For creating MockReader of innput stream. - * @author dybdahl + * @author dybis */ public class MockFeedReaderFactory extends FeedReaderFactory { diff --git a/vespaclient-core/OWNERS b/vespaclient-core/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/vespaclient-core/OWNERS +++ b/vespaclient-core/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 01705141ee0..660d3326f49 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -7,7 +7,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-core</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespaclient/.gitignore b/vespaclient/.gitignore index 235d7663475..afee18213d5 100644 --- a/vespaclient/.gitignore +++ b/vespaclient/.gitignore @@ -6,4 +6,3 @@ target /tmp /pom.xml.build Makefile -Testing diff --git a/vespaclient/OWNERS b/vespaclient/OWNERS index 0e39145d8c3..123437e2758 100644 --- a/vespaclient/OWNERS +++ b/vespaclient/OWNERS @@ -1 +1 @@ -dybdahl +dybis diff --git a/vespaclient/src/perl/bin/GetClusterState.pl b/vespaclient/src/perl/bin/GetClusterState.pl index 2352a5a0ca6..45af63c564c 100755 --- a/vespaclient/src/perl/bin/GetClusterState.pl +++ b/vespaclient/src/perl/bin/GetClusterState.pl @@ -1,9 +1,10 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -62,12 +63,12 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); use strict; use warnings; -use lib '$VESPA_HOME/lib/perl5/site_perl'; use Yahoo::Vespa::Bin::GetClusterState; diff --git a/vespaclient/src/perl/bin/GetNodeState.pl b/vespaclient/src/perl/bin/GetNodeState.pl index d373eadb65b..c3865e92732 100755 --- a/vespaclient/src/perl/bin/GetNodeState.pl +++ b/vespaclient/src/perl/bin/GetNodeState.pl @@ -1,9 +1,10 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -62,12 +63,12 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); use strict; use warnings; -use lib '$VESPA_HOME/lib/perl5/site_perl'; use Yahoo::Vespa::Bin::GetNodeState; diff --git a/vespaclient/src/perl/bin/SetNodeState.pl b/vespaclient/src/perl/bin/SetNodeState.pl index 7002ab523b5..6dc099f8e5f 100755 --- a/vespaclient/src/perl/bin/SetNodeState.pl +++ b/vespaclient/src/perl/bin/SetNodeState.pl @@ -1,9 +1,10 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section # Do not edit between here and END as this section should stay identical in all scripts +use warnings; use File::Basename; use File::Path; @@ -62,6 +63,7 @@ my $VESPA_HOME = $ENV{'VESPA_HOME'}; # END perl environment bootstrap section use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl'; use Yahoo::Vespa::Defaults; readConfFile(); diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 38028810afb..0282cc94ff6 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespajlib</artifactId> <packaging>container-plugin</packaging> diff --git a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java index 001386cd4b0..6c7a265ca60 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java @@ -30,7 +30,7 @@ import com.yahoo.text.DoubleParser; * s=40.183868,w=-74.819519,n=40.248291,e=-74.728798 * </pre> * - * @author Arne J + * @author arnej27959 */ public class BoundingBoxParser { diff --git a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java index 540f8300f95..5a1f6c249f2 100644 --- a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java +++ b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java @@ -12,7 +12,7 @@ import java.util.stream.Collectors; /** * Utilities for returning localhost addresses on Linux. - * See + * See * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037 * on why this is necessary. * @@ -20,35 +20,40 @@ import java.util.stream.Collectors; */ public class LinuxInetAddress { - private static Logger log = Logger.getLogger(LinuxInetAddress.class.getName()); - /** * Returns an InetAddress representing the address of the localhost. * A non-loopback address is preferred if available. - * IPv4 is preferred over IPv6 if available. + * An address that resolves to a hostname is preferred among non-loopback addresses. + * IPv4 is preferred over IPv6 among resolving addresses. * * @return a localhost address - * @throws UnknownHostException if an address could not be determined */ - public static InetAddress getLocalHost() throws UnknownHostException { - InetAddress localAddress; + public static InetAddress getLocalHost() { + InetAddress fallback = InetAddress.getLoopbackAddress(); try { - localAddress = InetAddress.getLocalHost(); + InetAddress localAddress = InetAddress.getLocalHost(); + fallback = localAddress; + List<InetAddress> nonLoopback = + getAllLocalFromNetwork().stream().filter(a -> ! a.isLoopbackAddress()).collect(Collectors.toList()); + if (nonLoopback.isEmpty()) { + return fallback; + } + fallback = nonLoopback.get(0); + List<InetAddress> resolving = + nonLoopback.stream().filter(a -> doesResolve(a)).collect(Collectors.toList()); + if (resolving.isEmpty()) { + return fallback; + } + fallback = resolving.get(0); + List<InetAddress> ipV4 = + resolving.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); + if (ipV4.isEmpty()) { + return fallback; + } + return ipV4.get(0); } catch (UnknownHostException e) { - return InetAddress.getLoopbackAddress(); + return fallback; } - - if ( ! localAddress.isLoopbackAddress()) return localAddress; - - List<InetAddress> nonLoopbackAddresses = - getAllLocalFromNetwork().stream().filter(a -> ! a.isLoopbackAddress()).collect(Collectors.toList()); - if (nonLoopbackAddresses.isEmpty()) return localAddress; - - List<InetAddress> ipV4NonLoopbackAddresses = - nonLoopbackAddresses.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); - if ( ! ipV4NonLoopbackAddresses.isEmpty()) return ipV4NonLoopbackAddresses.get(0); - - return nonLoopbackAddresses.get(0); } /** @@ -81,4 +86,10 @@ public class LinuxInetAddress { } } + private static boolean doesResolve(InetAddress addr) { + String asAddr = addr.getHostAddress(); + String asName = addr.getCanonicalHostName(); + return ! asAddr.equals(asName); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java index f924740321f..1c1b818fb36 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java @@ -15,14 +15,14 @@ public class ForceLoad { * @param classNames array of names of classes (without package prefix) * to force load. **/ - public static void forceLoad(String packageName, String[] classNames) + public static void forceLoad(String packageName, String[] classNames, ClassLoader loader) throws ForceLoadError { String fullClassName = ""; try { for (String className : classNames) { fullClassName = packageName + "." + className; - Class.forName(fullClassName); + Class.forName(fullClassName, true, loader); } } catch (Exception e) { throw new ForceLoadError(fullClassName, e); diff --git a/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java b/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java index 47a8ade2235..b09296389b8 100644 --- a/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.geo; /** * Tests for the BoundingBoxParser class. * - * @author Arne J + * @author arnej27959 */ public class BoundingBoxParserTestCase extends junit.framework.TestCase { diff --git a/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java b/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java index ec4f716247e..078142014cd 100644 --- a/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java @@ -8,8 +8,10 @@ public class ForceLoadTestCase extends junit.framework.TestCase { } public void testLoadClasses() { + try { - ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar" }); + ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar" }, + this.getClass().getClassLoader()); } catch (ForceLoadError e) { e.printStackTrace(); assertTrue(false); @@ -18,7 +20,8 @@ public class ForceLoadTestCase extends junit.framework.TestCase { public void testLoadBogusClass() { try { - ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar", "Baz" }); + ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar", "Baz" }, + this.getClass().getClassLoader()); } catch (ForceLoadError e) { return; } diff --git a/vespalib/.gitignore b/vespalib/.gitignore index 727ed8ec9e8..65e000de4f0 100644 --- a/vespalib/.gitignore +++ b/vespalib/.gitignore @@ -2,4 +2,3 @@ bin doc lib Makefile -Testing diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index e4c9ab27225..4af66d89ed5 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -54,7 +54,6 @@ vespa_define_module( src/tests/net/socket src/tests/objects/nbostream src/tests/optimized - src/tests/placement-delete src/tests/printable src/tests/priority_queue src/tests/random diff --git a/vespalib/src/tests/barrier/.gitignore b/vespalib/src/tests/barrier/.gitignore index c34ea278112..f15a24ef252 100644 --- a/vespalib/src/tests/barrier/.gitignore +++ b/vespalib/src/tests/barrier/.gitignore @@ -1,2 +1 @@ vespalib_barrier_test_app -Testing diff --git a/vespalib/src/tests/box/.gitignore b/vespalib/src/tests/box/.gitignore index 76ae8a62fd2..d3a9d8db21c 100644 --- a/vespalib/src/tests/box/.gitignore +++ b/vespalib/src/tests/box/.gitignore @@ -1,2 +1 @@ vespalib_box_test_app -Testing diff --git a/vespalib/src/tests/dual_merge_director/.gitignore b/vespalib/src/tests/dual_merge_director/.gitignore index b190240063a..6d65b032c84 100644 --- a/vespalib/src/tests/dual_merge_director/.gitignore +++ b/vespalib/src/tests/dual_merge_director/.gitignore @@ -1,2 +1 @@ vespalib_dual_merge_director_test_app -Testing diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index 0eb08a80f41..5deab82b6fc 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -116,4 +116,45 @@ TEST_F("requireThatNewTasksAreDroppedAfterShutdown", MyState()) { TEST_DO(f1.open().shutdown().execute(5).sync().check(5, 0, 0, 0)); } + +struct WaitTask : public Executor::Task { + Gate &gate; + WaitTask(Gate &g) : gate(g) {} + virtual void run() { gate.await(); } +}; + +struct WaitState { + ThreadStackExecutor executor; + std::vector<Gate> block_task; + std::vector<Gate> wait_done; + WaitState(size_t num_threads) + : executor(num_threads / 2, 128000), block_task(num_threads - 2), wait_done(num_threads - 1) + { + for (auto &gate: block_task) { + auto result = executor.execute(std::make_unique<WaitTask>(gate)); + ASSERT_TRUE(result.get() == nullptr); + } + } + void wait(size_t count) { + executor.wait_for_task_count(count); + wait_done[count].countDown(); + } +}; + +TEST_MT_F("require that threads can wait for a specific task count", 7, WaitState(num_threads)) { + if (thread_id == 0) { + for (size_t next_done = (num_threads - 2); next_done-- > 0;) { + if (next_done < f1.block_task.size()) { + f1.block_task[f1.block_task.size() - 1 - next_done].countDown(); + } + EXPECT_TRUE(f1.wait_done[next_done].await(25000)); + for (size_t i = 0; i < next_done; ++i) { + EXPECT_TRUE(!f1.wait_done[i].await(20)); + } + } + } else { + f1.wait(thread_id - 1); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/placement-delete/.gitignore b/vespalib/src/tests/placement-delete/.gitignore deleted file mode 100644 index fbf1f708fc0..00000000000 --- a/vespalib/src/tests/placement-delete/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.depend -Makefile -fail -hello -placement-delete_test -undef -vespalib_placement-delete_test_app diff --git a/vespalib/src/tests/placement-delete/CMakeLists.txt b/vespalib/src/tests/placement-delete/CMakeLists.txt deleted file mode 100644 index 74c4a431958..00000000000 --- a/vespalib/src/tests/placement-delete/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_placement-delete_test_app TEST - SOURCES - placement-delete.cpp - DEPENDS - vespalib -) - -vespa_add_test( - NAME vespalib_placement-delete_test_app - NO_VALGRIND COMMAND vespalib_placement-delete_test_app - ENVIRONMENT "CXX_PROG=g++" -) diff --git a/vespalib/src/tests/placement-delete/DESC b/vespalib/src/tests/placement-delete/DESC deleted file mode 100644 index 49ef1e5fb3c..00000000000 --- a/vespalib/src/tests/placement-delete/DESC +++ /dev/null @@ -1,4 +0,0 @@ -This test checks whether gcc generates code to invoke placement delete -when a templated placement new throws an exception. We expect this -test to fail until gcc has been fixed to handle templated placement -new correctly. diff --git a/vespalib/src/tests/placement-delete/FILES b/vespalib/src/tests/placement-delete/FILES deleted file mode 100644 index 6214cb9fb02..00000000000 --- a/vespalib/src/tests/placement-delete/FILES +++ /dev/null @@ -1,4 +0,0 @@ -placement-delete.cpp -hello.cpp -fail.cpp -undef.cpp diff --git a/vespalib/src/tests/placement-delete/placement-delete.cpp b/vespalib/src/tests/placement-delete/placement-delete.cpp deleted file mode 100644 index 0d1cb200110..00000000000 --- a/vespalib/src/tests/placement-delete/placement-delete.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> -#include <vespa/vespalib/util/stringfmt.h> - -using namespace vespalib; - -bool checkCompile(const std::string &base) { - std::string out; - std::string gcc = getenv("CXX_PROG"); // TODO: override from environment - std::string cmd = make_string("%s -o %s %s.cpp 2>&1", gcc.c_str(), base.c_str(), base.c_str()); - bool ok = SlaveProc::run(cmd.c_str(), out); - fprintf(stderr, "CMD: %s\n(compile output follows...)\n%s\n", cmd.c_str(), out.c_str()); - return ok; -} - -TEST("require that valid test program can be compiled") { - EXPECT_EQUAL(checkCompile("hello"), true); -} - -TEST("require that bogus test program can not be compiled") { - EXPECT_EQUAL(checkCompile("fail"), false); -} - -TEST("require that templated placement delete is instantiated resulting in a compile error") { - EXPECT_EQUAL(checkCompile("undef"), false); -} - -TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/placement-delete/undef.cpp b/vespalib/src/tests/placement-delete/undef.cpp deleted file mode 100644 index c6ba20534f8..00000000000 --- a/vespalib/src/tests/placement-delete/undef.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -// This is a compile-time test which should fail because undef<T> -// cannot be instantiated - -#include <stdlib.h> - -template <class T> class undef; - -struct A { - A() { throw 1; } -}; - -template<typename T> class Pool { }; - -template<typename T> -inline void *operator new(size_t size,Pool<T>& pool) -{ - return malloc(size); -} - -template<typename T> -inline void operator delete(void *p,Pool<T>& pool) -{ - undef<T> t; - free(p); -} - -int main () -{ - Pool<int> pool; - new (pool) A(); - return 0; -} diff --git a/vespalib/src/tests/slaveproc/slaveproc_test.cpp b/vespalib/src/tests/slaveproc/slaveproc_test.cpp index 95f8021f7d2..71427be6709 100644 --- a/vespalib/src/tests/slaveproc/slaveproc_test.cpp +++ b/vespalib/src/tests/slaveproc/slaveproc_test.cpp @@ -13,7 +13,7 @@ TEST("simple run, ignore output, failure") { } TEST("simple run, ignore output, timeout") { - EXPECT_TRUE(!SlaveProc::run("sleep 60", 10)); + EXPECT_TRUE(!SlaveProc::run("exec sleep 60", 10)); } TEST("simple run") { @@ -58,16 +58,17 @@ TEST("simple run with input, don't strip multi-line output") { TEST_MT("simple run, partial output due to timeout", 2) { std::string out; std::vector<size_t> timeouts({150, 300, 3000, 6000, 60000}); + const char *my_cmd = "exec perl -e '$| = 1; print \"foo\\\n\"; sleep(600); print \"bar\\\n\"'"; for (size_t timeout: timeouts) { fprintf(stderr, "... verifying partial output with%s input (timeout = %zu)\n", (thread_id == 0) ? "out" : "", timeout); if (thread_id == 0) { out.clear(); - EXPECT_TRUE(!SlaveProc::run("echo foo; sleep 600; echo bar", out, timeout)); + EXPECT_TRUE(!SlaveProc::run(my_cmd, out, timeout)); } else { out.clear(); std::string in = "ignored\n"; - EXPECT_TRUE(!SlaveProc::run(in, "echo foo; sleep 600; echo bar", out, timeout)); + EXPECT_TRUE(!SlaveProc::run(in, my_cmd, out, timeout)); } if (out == "foo") { break; diff --git a/vespalib/src/tests/testkit-subset/.gitignore b/vespalib/src/tests/testkit-subset/.gitignore index 1007cdc2b36..fc843d3776f 100644 --- a/vespalib/src/tests/testkit-subset/.gitignore +++ b/vespalib/src/tests/testkit-subset/.gitignore @@ -1,2 +1,3 @@ /out.txt vespalib_testkit-subset_test_app +/out.relpath.txt diff --git a/vespalib/src/tests/testkit-subset/out.ref.2.txt b/vespalib/src/tests/testkit-subset/out.ref.2.txt new file mode 100644 index 00000000000..b08880669f7 --- /dev/null +++ b/vespalib/src/tests/testkit-subset/out.ref.2.txt @@ -0,0 +1,20 @@ +testkit-subset_test.cpp: info: running test suite 'testkit-subset_test.cpp' +testkit-subset_test.cpp: info: only running tests matching 'pass' +testkit-subset_test.cpp: info: trace: thread '0(1)' (testkit-subset_test.cpp:5) +testkit-subset_test.cpp: info: status_for_test 'will pass main': PASS +testkit-subset_test.cpp: info: trace: thread '0(1)' (testkit-subset_extra.cpp:5) +testkit-subset_test.cpp: info: status_for_test 'will pass extra': PASS +testkit-subset_test.cpp: info: test summary --- 2 test(s) passed --- 0 test(s) failed +testkit-subset_test.cpp: info: test summary --- 2 test(s) skipped +testkit-subset_test.cpp: info: imported 2 passed check(s) from 1 thread(s) +testkit-subset_test.cpp: info: summary --- 2 check(s) passed --- 0 check(s) failed +testkit-subset_test.cpp: info: CONCLUSION: PASS +testkit-subset_test.cpp: info: running test suite 'testkit-subset_test.cpp' +testkit-subset_test.cpp: info: only running tests matching 'extra\.cpp:.*pass.*' +testkit-subset_test.cpp: info: trace: thread '0(1)' (testkit-subset_extra.cpp:5) +testkit-subset_test.cpp: info: status_for_test 'will pass extra': PASS +testkit-subset_test.cpp: info: test summary --- 1 test(s) passed --- 0 test(s) failed +testkit-subset_test.cpp: info: test summary --- 3 test(s) skipped +testkit-subset_test.cpp: info: imported 1 passed check(s) from 1 thread(s) +testkit-subset_test.cpp: info: summary --- 1 check(s) passed --- 0 check(s) failed +testkit-subset_test.cpp: info: CONCLUSION: PASS diff --git a/vespalib/src/tests/testkit-subset/testkit-subset_test.sh b/vespalib/src/tests/testkit-subset/testkit-subset_test.sh index 6da1584a3f3..8d6556f30a8 100755 --- a/vespalib/src/tests/testkit-subset/testkit-subset_test.sh +++ b/vespalib/src/tests/testkit-subset/testkit-subset_test.sh @@ -3,4 +3,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/" TEST_SUBSET=pass $VALGRIND ./vespalib_testkit-subset_test_app 2> out.txt TEST_SUBSET="extra\.cpp:.*pass.*" $VALGRIND ./vespalib_testkit-subset_test_app 2>> out.txt cat out.txt | grep "\.cpp: " | sed "s~$DIR~~g" > out.relpath.txt +cmp -s out.relpath.txt out.ref.2.txt && exit 0 diff -u out.relpath.txt out.ref.txt diff --git a/vespalib/src/tests/text/lowercase/to-c-code.pl b/vespalib/src/tests/text/lowercase/to-c-code.pl index 63a87fb56ae..8fa00992e29 100755 --- a/vespalib/src/tests/text/lowercase/to-c-code.pl +++ b/vespalib/src/tests/text/lowercase/to-c-code.pl @@ -1,10 +1,9 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # input looks like: # lowercase( 65 )= 97 - my %lowercase; my %blocks; diff --git a/vespalib/src/tests/tutorial/CMakeLists.txt b/vespalib/src/tests/tutorial/CMakeLists.txt index 40f12c42edb..120c848b2ae 100644 --- a/vespalib/src/tests/tutorial/CMakeLists.txt +++ b/vespalib/src/tests/tutorial/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_executable(vespalib_make_tutorial_app TEST DEPENDS vespalib ) -vespa_add_test(NAME vespalib_make_tutorial_app COMMAND vespalib_make_tutorial_app) +vespa_add_test(NAME vespalib_make_tutorial_app COMMAND sh ./compare-tutorials.sh) vespa_add_executable(vespalib_xml_escape_app SOURCES xml_escape.cpp diff --git a/vespalib/src/tests/tutorial/checks/.gitignore b/vespalib/src/tests/tutorial/checks/.gitignore index bd3d3565773..6cc70e1c25a 100644 --- a/vespalib/src/tests/tutorial/checks/.gitignore +++ b/vespalib/src/tests/tutorial/checks/.gitignore @@ -1,2 +1 @@ vespalib_checks_test_app -Testing diff --git a/vespalib/src/tests/tutorial/compare-tutorials.sh b/vespalib/src/tests/tutorial/compare-tutorials.sh new file mode 100755 index 00000000000..2c2aa7f4c44 --- /dev/null +++ b/vespalib/src/tests/tutorial/compare-tutorials.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +./vespalib_make_tutorial_app > tutorial_out.html +diff -u tutorial.html tutorial_out.html || true +echo "IGNORED: vespalib_make_tutorial_app" diff --git a/vespalib/src/tests/tutorial/fixtures/.gitignore b/vespalib/src/tests/tutorial/fixtures/.gitignore index 049a39ac736..98e7131620a 100644 --- a/vespalib/src/tests/tutorial/fixtures/.gitignore +++ b/vespalib/src/tests/tutorial/fixtures/.gitignore @@ -1,2 +1 @@ vespalib_fixtures_test_app -Testing diff --git a/vespalib/src/tests/tutorial/make_example.sh b/vespalib/src/tests/tutorial/make_example.sh index 9b015c93e2b..b5919254bc9 100755 --- a/vespalib/src/tests/tutorial/make_example.sh +++ b/vespalib/src/tests/tutorial/make_example.sh @@ -13,7 +13,7 @@ echo "<pre class=\"prettyprint linenums\">" (cd $dirname && cat $filename) | ./vespalib_xml_escape_app echo "</pre>" echo "<pre class=\"output\">" -(cd $dirname && make all > /dev/null 2>&1) -(cd $dirname && make test 2>&1) | ./vespalib_xml_escape_app +DIRNAME=`(cd $dirname && /bin/pwd)` +(cd $dirname && ./vespalib_${filename%.cpp}_app 2>&1) | perl -pe "s{$DIRNAME/}{}g" | ./vespalib_xml_escape_app echo "</pre>" echo "</div>" diff --git a/vespalib/src/tests/tutorial/minimal/.gitignore b/vespalib/src/tests/tutorial/minimal/.gitignore index bb09f68684f..48d49394413 100644 --- a/vespalib/src/tests/tutorial/minimal/.gitignore +++ b/vespalib/src/tests/tutorial/minimal/.gitignore @@ -1,2 +1 @@ vespalib_minimal_test_app -Testing diff --git a/vespalib/src/tests/tutorial/simple/.gitignore b/vespalib/src/tests/tutorial/simple/.gitignore index 42594074945..aa48ba3f011 100644 --- a/vespalib/src/tests/tutorial/simple/.gitignore +++ b/vespalib/src/tests/tutorial/simple/.gitignore @@ -1,2 +1 @@ vespalib_simple_test_app -Testing diff --git a/vespalib/src/tests/tutorial/threads/.gitignore b/vespalib/src/tests/tutorial/threads/.gitignore index a90822b0e7c..7668f1b3384 100644 --- a/vespalib/src/tests/tutorial/threads/.gitignore +++ b/vespalib/src/tests/tutorial/threads/.gitignore @@ -1,2 +1 @@ vespalib_threads_test_app -Testing diff --git a/vespalib/src/tests/tutorial/tutorial.html b/vespalib/src/tests/tutorial/tutorial.html index 744dd9541e2..8b6706566f0 100644 --- a/vespalib/src/tests/tutorial/tutorial.html +++ b/vespalib/src/tests/tutorial/tutorial.html @@ -190,7 +190,6 @@ checks_test.cpp: info: test summary --- 1 test(s) passed --- 1 test(s) failed checks_test.cpp: info: imported 11 passed check(s) from 1 thread(s) checks_test.cpp: info: summary --- 11 check(s) passed --- 1 check(s) failed checks_test.cpp: info: CONCLUSION: FAIL -make: *** [test] Error 1 </pre> </div> diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index d72797fe869..2eeffce2832 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -204,6 +204,7 @@ File::open(int flags, bool autoCreateDirectories) { VESPA_STRLOC); } _flags = flags; + if (_close && _fd != -1) close(); _fd = fd; LOG(debug, "open(%s, %d). File opened with file descriptor %d.", _filename.c_str(), flags, fd); @@ -335,6 +336,13 @@ File::read(void *buf, size_t bufsize, off_t offset) const remaining -= bytesread; buf = ((char*) buf) + bytesread; offset += bytesread; + if (((_flags & DIRECTIO) != 0) && ((bytesread % 512) != 0) && + (offset == getFileSize())) { + LOG(spam, "read(%s): Found EOF. Directio read to unaligned " + "file end at offset %" PRIu64 ".", + _filename.c_str(), offset); + break; + } } else if (bytesread == 0) { // EOF LOG(spam, "read(%s): Found EOF. Zero bytes read from offset %" PRIu64 ".", diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index 7ab9f56b5be..8f34beec535 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -259,7 +259,7 @@ TensorMapper::mapToDense(const Tensor &tensor, const TensorType &type) } std::unique_ptr<Tensor> -TensorMapper::map(const Tensor &tensor) +TensorMapper::map(const Tensor &tensor) const { switch (_type.type()) { case TensorType::Type::INVALID: diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h index f6eb087bc60..36ce0426c69 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.h @@ -35,7 +35,7 @@ public: static std::unique_ptr<Tensor> mapToDense(const Tensor &tensor, const TensorType &type); - std::unique_ptr<Tensor> map(const Tensor &tensor); + std::unique_ptr<Tensor> map(const Tensor &tensor) const; }; diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index bee1d9c17ec..780f0ed5dd0 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -6,6 +6,47 @@ namespace vespalib { void +ThreadStackExecutorBase::BlockedThread::wait() const +{ + MonitorGuard guard(monitor); + while (blocked) { + guard.wait(); + } +} + +void +ThreadStackExecutorBase::BlockedThread::unblock() +{ + MonitorGuard guard(monitor); + blocked = false; + guard.signal(); +} + +//----------------------------------------------------------------------------- + +void +ThreadStackExecutorBase::block_thread(const LockGuard &, BlockedThread &blocked_thread) +{ + auto pos = _blocked.begin(); + while ((pos != _blocked.end()) && + ((*pos)->wait_task_count < blocked_thread.wait_task_count)) + { + ++pos; + } + _blocked.insert(pos, &blocked_thread); +} + +void +ThreadStackExecutorBase::unblock_threads(const MonitorGuard &) +{ + while (!_blocked.empty() && (_taskCount <= _blocked.back()->wait_task_count)) { + BlockedThread &blocked_thread = *(_blocked.back()); + _blocked.pop_back(); + blocked_thread.unblock(); + } +} + +void ThreadStackExecutorBase::assignTask(const TaggedTask &task, Worker &worker) { MonitorGuard monitor(worker.monitor); @@ -26,6 +67,7 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) _barrier.completeEvent(worker.task.token); worker.task.task = 0; } + unblock_threads(monitor); if (!_tasks.empty()) { worker.task = _tasks.front(); _tasks.pop(); @@ -168,6 +210,19 @@ ThreadStackExecutorBase::sync() } void +ThreadStackExecutorBase::wait_for_task_count(uint32_t task_count) +{ + LockGuard lock(_monitor); + if (_taskCount <= task_count) { + return; + } + BlockedThread self(task_count); + block_thread(lock, self); + lock.unlock(); // <- UNLOCK + self.wait(); +} + +void ThreadStackExecutorBase::cleanup() { shutdown().sync(); @@ -179,6 +234,7 @@ ThreadStackExecutorBase::~ThreadStackExecutorBase() { assert(_pool.isClosed()); assert(_taskCount == 0); + assert(_blocked.empty()); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index f66cd74dae1..215a8377fc6 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -52,17 +52,31 @@ private: void completeBarrier() { gate.countDown(); } }; + struct BlockedThread { + const uint32_t wait_task_count; + Monitor monitor; + bool blocked; + BlockedThread(uint32_t wait_task_count_in) + : wait_task_count(wait_task_count_in), monitor(), blocked(true) {} + void wait() const; + void unblock(); + }; + FastOS_ThreadPool _pool; Monitor _monitor; Stats _stats; Gate _executorCompletion; ArrayQueue<TaggedTask> _tasks; ArrayQueue<Worker*> _workers; + std::vector<BlockedThread*> _blocked; EventBarrier<BarrierCompletion> _barrier; uint32_t _taskCount; uint32_t _taskLimit; bool _closed; + void block_thread(const LockGuard &, BlockedThread &blocked_thread); + void unblock_threads(const MonitorGuard &); + /** * Assign the given task to the given idle worker. This will wake * up a worker thread that is blocked in the obtainTask function. @@ -153,6 +167,14 @@ public: virtual ThreadStackExecutorBase &sync(); /** + * Block the calling thread until the current task count is equal + * to or lower than the given value. + * + * @param task_count target value to wait for + **/ + void wait_for_task_count(uint32_t task_count); + + /** * Shut down this executor. This will make this executor reject * all new tasks. * diff --git a/vespalog/.gitignore b/vespalog/.gitignore index ad5e60bcc0f..efdf6e430c8 100644 --- a/vespalog/.gitignore +++ b/vespalog/.gitignore @@ -12,4 +12,3 @@ testLogs vespalog.iml /pom.xml.build Makefile -Testing diff --git a/vespalog/pom.xml b/vespalog/pom.xml index 5a39f43b1c3..e25d4e66a56 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespalog</artifactId> <version>6-SNAPSHOT</version> diff --git a/vespalog/src/logfmt/logfmt.pl b/vespalog/src/logfmt/logfmt.pl index 9152d1fcfec..8d01feb3a6b 100644..100755 --- a/vespalog/src/logfmt/logfmt.pl +++ b/vespalog/src/logfmt/logfmt.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/usr/bin/env perl # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN perl environment bootstrap section diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp index fde952720c7..9ffd2d09a1e 100644 --- a/vespalog/src/logger/runserver.cpp +++ b/vespalog/src/logger/runserver.cpp @@ -314,6 +314,8 @@ int main(int argc, char *argv[]) const char *pidfile = "vespa-runserver.pid"; // XXX bad default? const char *killcmd = NULL; + signal(SIGQUIT, SIG_IGN); + int ch; while ((ch = getopt(argc, argv, "k:s:r:p:Sh")) != -1) { switch (ch) { diff --git a/vespamalloc/.gitignore b/vespamalloc/.gitignore index e273b5725de..08001c737d7 100644 --- a/vespamalloc/.gitignore +++ b/vespamalloc/.gitignore @@ -1,3 +1,2 @@ test Makefile -Testing diff --git a/vespamalloc/src/vespamalloc/util/osmem.cpp b/vespamalloc/src/vespamalloc/util/osmem.cpp index c5f44342c75..0b02900b678 100644 --- a/vespamalloc/src/vespamalloc/util/osmem.cpp +++ b/vespamalloc/src/vespamalloc/util/osmem.cpp @@ -6,6 +6,7 @@ #include <sys/mman.h> #include <linux/mman.h> #include <algorithm> +#include <errno.h> namespace vespamalloc { diff --git a/vsm/.gitignore b/vsm/.gitignore index 2f5c05fd2c2..3969348a427 100644 --- a/vsm/.gitignore +++ b/vsm/.gitignore @@ -3,4 +3,3 @@ lib /target /pom.xml.build Makefile -Testing diff --git a/vsm/pom.xml b/vsm/pom.xml index f0932d0ffc6..5962dc5f61d 100644 --- a/vsm/pom.xml +++ b/vsm/pom.xml @@ -8,7 +8,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vsm</artifactId> <version>6-SNAPSHOT</version> diff --git a/yolean/pom.xml b/yolean/pom.xml index c832f5e7f46..75809187eba 100644 --- a/yolean/pom.xml +++ b/yolean/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>yolean</artifactId> <version>6-SNAPSHOT</version> diff --git a/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java b/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java new file mode 100644 index 00000000000..204dcaacf28 --- /dev/null +++ b/yolean/src/main/java/com/yahoo/yolean/system/CatchSigTerm.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.yolean.system; + +import java.lang.reflect.*; + +// import sun.misc.Signal; +// import sun.misc.SignalHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class CatchSigTerm { + /** + * Sets up a signal handler for SIGTERM, where a given AtomicBoolean + * gets a true value when the TERM signal is caught. + * + * Callers basically have two options for acting on the TERM signal: + * + * They may choose to synchronize and wait() on this variable, + * and they will be notified when it changes state to true. To avoid + * problems with spurious wakeups, use a while loop and wait() + * again if the state is still false. As soon as the caller has been + * woken up and the state is true, the application should exit as + * soon as possible. + * + * They may also choose to poll the state of this variable. As soon + * as its state becomes true, the signal has been received, and the + * application should exit as soon as possible. + * + * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. + */ + @SuppressWarnings("rawtypes") + public static void setup(final AtomicBoolean signalCaught) { + signalCaught.set(false); + try { + Class shc = Class.forName("sun.misc.SignalHandler"); + Class ssc = Class.forName("sun.misc.Signal"); + + InvocationHandler ihandler = new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + synchronized (signalCaught) { + signalCaught.set(true); + signalCaught.notifyAll(); + } + return null; + } + }; + Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), + new Class[] { shc }, + ihandler); + Constructor[] c = ssc.getDeclaredConstructors(); + assert c.length == 1; + Object sigterm = c[0].newInstance("TERM"); + Method m = findMethod(ssc, "handle"); + assert m != null; // "NoSuchMethodException" + m.invoke(null, sigterm, shandler); + } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + System.err.println("FAILED setting up signal catching: "+e); + } + } + + private static Method findMethod(Class<?> c, String name) { + for (Method m : c.getDeclaredMethods()) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } +} diff --git a/vespalib/src/tests/placement-delete/hello.cpp b/yolean/src/main/java/com/yahoo/yolean/system/package-info.java index aeab3a27762..e727474ec52 100644 --- a/vespalib/src/tests/placement-delete/hello.cpp +++ b/yolean/src/main/java/com/yahoo/yolean/system/package-info.java @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdio.h> +@ExportPackage +package com.yahoo.yolean.system; -int main () { - fprintf(stdout, "Hello World!\n"); - return 0; -} +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java b/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java new file mode 100644 index 00000000000..d2ad84948e1 --- /dev/null +++ b/yolean/src/test/java/com/yahoo/yolean/system/CatchSigTermTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.yolean.system; + +import com.yahoo.yolean.system.CatchSigTerm; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author arnej27959 + */ +public class CatchSigTermTestCase extends junit.framework.TestCase { + + public CatchSigTermTestCase(String name) { + super(name); + } + + public void testThatSetupCompiles() { + CatchSigTerm.setup(new AtomicBoolean(false)); + } +} diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index d35d1f6b006..46070faa23a 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -6,7 +6,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> <version>6-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zkfacade</artifactId> <packaging>container-plugin</packaging> |