From c56c752cf29c6db497c1dc3640ab134fbd3274a3 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Thu, 30 Jun 2016 16:22:38 +0200 Subject: Add basic blog-recommendation sample app --- sample-apps/blog-recommendation/README.md | 55 +++++++++++++ sample-apps/blog-recommendation/pom.xml | 85 +++++++++++++++++++++ .../src/main/application/deployment.xml | 5 ++ .../search/query-profiles/types/root.xml | 3 + .../application/searchdefinitions/blog_post.sd | 66 ++++++++++++++++ .../src/main/application/searchdefinitions/user.sd | 26 +++++++ .../src/main/application/services.xml | 36 +++++++++ .../java/com/yahoo/example/BlogTensorSearcher.java | 43 +++++++++++ .../com/yahoo/example/UserProfileSearcher.java | 89 ++++++++++++++++++++++ 9 files changed, 408 insertions(+) create mode 100644 sample-apps/blog-recommendation/README.md create mode 100644 sample-apps/blog-recommendation/pom.xml create mode 100644 sample-apps/blog-recommendation/src/main/application/deployment.xml create mode 100644 sample-apps/blog-recommendation/src/main/application/search/query-profiles/types/root.xml create mode 100644 sample-apps/blog-recommendation/src/main/application/searchdefinitions/blog_post.sd create mode 100644 sample-apps/blog-recommendation/src/main/application/searchdefinitions/user.sd create mode 100644 sample-apps/blog-recommendation/src/main/application/services.xml create mode 100644 sample-apps/blog-recommendation/src/main/java/com/yahoo/example/BlogTensorSearcher.java create mode 100644 sample-apps/blog-recommendation/src/main/java/com/yahoo/example/UserProfileSearcher.java diff --git a/sample-apps/blog-recommendation/README.md b/sample-apps/blog-recommendation/README.md new file mode 100644 index 00000000000..0b6a00c9743 --- /dev/null +++ b/sample-apps/blog-recommendation/README.md @@ -0,0 +1,55 @@ +Basic Search Application +================== +Start by [deploying a sample application](http://vespa.corp.yahoo.com/6/documentation/developing-with-vespa.html). + +### Find the endpoint + +When you have successfully deployed your own compiled version of the application above, you need to find the name of the "endpoint". +The endpoint is used for feeding and searching for data. +**Please allow a few minutes for the endpoint to appear after deployment** +You can find this endpoint by doing: + ```sh + + mvn vespa:endpoints | grep Endpoints + ``` + +You can also find it by looking at the [Hosted Vespa Dashboard](http://dashboard.vespa.corp.yahoo.com). + + +### Feed and search + 1. **Feed** the data that is to be searched + ```sh + + # Feeding two documents + curl -X POST --data-binary @music-data-1.json /document/v1/music/music/docid/1 | python -m json.tool + curl -X POST --data-binary @music-data-2.json /document/v1/music/music/docid/2 | python -m json.tool + + ``` + +For feeding many documents fast and reliable, checkout [feeding example](https://git.corp.yahoo.com/vespa-samples/basic-feeding-client) + + 2. **Visit documents + + Since we do not have many documents we can list them all + ```sh + + # All documents + curl /document/v1/music/music/docid | python -m json.tool + + # Document with id 1 + curl /document/v1/music/music/docid/1 | python -m json.tool + + ``` + + 3. **Search** + We can also search for documents: + ```sh + + curl '/search/?query=bad' | python -m json.tool + + + ``` + +### Next step: from development to production +See [continuous deployments](http://vespa.corp.yahoo.com/6/documentation/continuous-deployment.html) for how to implement continuous deployments for production. +See [RESTified Document Operation API](http://vespa.corp.yahoo.com/6/documentation/document_api_v1.html) for documentation about the REST API for document operations. diff --git a/sample-apps/blog-recommendation/pom.xml b/sample-apps/blog-recommendation/pom.xml new file mode 100644 index 00000000000..6e55e109f43 --- /dev/null +++ b/sample-apps/blog-recommendation/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + com.yahoo.example + recommendation + container-plugin + 0.0.1 + application + + + UTF-8 + true + blog-recommendation + default + + + + com.yahoo.vespa.tenant + base + RELEASE + + + + + ymaven + http://ymaven.corp.yahoo.com:9999/proximity/repository/public + + false + + + + maven2-repository.dev.java.net + http://download.java.net/maven/2 + + false + + + + vespa-maven-release + vespa-maven-release + http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local + + true + + + false + + + + + + ymaven + http://ymaven.corp.yahoo.com:9999/proximity/repository/public + + false + + + + vespa-maven-release + vespa-maven-release + http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local + + true + + + false + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all + -Werror + + + + + + diff --git a/sample-apps/blog-recommendation/src/main/application/deployment.xml b/sample-apps/blog-recommendation/src/main/application/deployment.xml new file mode 100644 index 00000000000..0a2397bbca9 --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/application/deployment.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/sample-apps/blog-recommendation/src/main/application/search/query-profiles/types/root.xml b/sample-apps/blog-recommendation/src/main/application/search/query-profiles/types/root.xml new file mode 100644 index 00000000000..6362bbe48ce --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/application/search/query-profiles/types/root.xml @@ -0,0 +1,3 @@ + + + diff --git a/sample-apps/blog-recommendation/src/main/application/searchdefinitions/blog_post.sd b/sample-apps/blog-recommendation/src/main/application/searchdefinitions/blog_post.sd new file mode 100644 index 00000000000..1b92822e425 --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/application/searchdefinitions/blog_post.sd @@ -0,0 +1,66 @@ +search blog_post { + document blog_post { + field date_gmt type string { + indexing: summary + } + field language type string { + indexing: summary + } + + field author type string { + indexing: summary + } + + field url type string { + indexing: summary + } + + field title type string { + indexing: summary | index + } + + field blog type string { + indexing: summary + } + + field post_id type string { + indexing: summary + } + + field tags type array { + indexing: summary + } + + field blogname type string { + indexing: summary + } + + field content type string { + indexing: summary | index + } + + field categories type array { + indexing: summary | index + } + + field user_item_cf type tensor { + indexing: summary | attribute + attribute: tensor(user_item_cf[10]) + } + + field has_user_item_cf type byte { + indexing: summary | attribute + attribute: fast-search + } + + } + + rank-profile tensor { + first-phase { + expression { + sum(query(user_item_cf) * attribute(user_item_cf)) + } + } + } + +} diff --git a/sample-apps/blog-recommendation/src/main/application/searchdefinitions/user.sd b/sample-apps/blog-recommendation/src/main/application/searchdefinitions/user.sd new file mode 100644 index 00000000000..0c2a6e50aa6 --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/application/searchdefinitions/user.sd @@ -0,0 +1,26 @@ +search user { + document user { + + field user_id type string { + indexing: summary | attribute + attribute: fast-search + } + + field has_read_items type array { + indexing: summary | attribute + } + + # see http://vespa.corp.yahoo.com/6/documentation/reference/search-definitions.html#tensor-type-spec + field user_item_cf type tensor { + indexing: summary | attribute + attribute: tensor(user_item_cf[10]) + } + + field has_user_item_cf type byte { + indexing: summary | attribute + attribute: fast-search + } + + } + +} diff --git a/sample-apps/blog-recommendation/src/main/application/services.xml b/sample-apps/blog-recommendation/src/main/application/services.xml new file mode 100644 index 00000000000..d1650eaca8c --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/application/services.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + 1 + + + + + diff --git a/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/BlogTensorSearcher.java b/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/BlogTensorSearcher.java new file mode 100644 index 00000000000..f9ff27c4813 --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/BlogTensorSearcher.java @@ -0,0 +1,43 @@ +package com.yahoo.example; + +import com.yahoo.prelude.query.IntItem; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.QueryTreeUtil; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.tensor.Tensor; + +public class BlogTensorSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + + Object userItemCfProperty = query.properties().get("user_item_cf"); + if (userItemCfProperty != null) { + + // Modify the query by restricting to blog_posts... + query.getModel().setRestrict("blog_post"); + + // ... that has a tensor field fed + QueryTreeUtil.andQueryItemWithRoot(query, new IntItem(1, "has_user_item_cf")); + + // Modify the ranking by using the 'tensor' rank-profile (as defined in blog_post.sd)... + query.properties().set(new CompoundName("ranking"), "tensor"); + + // ... and setting 'query(user_item_cf)' used in that rank-profile + query.getRanking().getFeatures().put("query(user_item_cf)", toTensor(userItemCfProperty)); + } + + return execution.search(query); + } + + private Tensor toTensor(Object tensor) { + if (tensor instanceof Tensor) { + return (Tensor) tensor; + } + return Tensor.from(tensor.toString()); + } + +} diff --git a/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/UserProfileSearcher.java b/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/UserProfileSearcher.java new file mode 100644 index 00000000000..63bf4110e67 --- /dev/null +++ b/sample-apps/blog-recommendation/src/main/java/com/yahoo/example/UserProfileSearcher.java @@ -0,0 +1,89 @@ +package com.yahoo.example; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.tensor.MapTensorBuilder; +import com.yahoo.tensor.Tensor; + +import java.util.Iterator; +import java.util.Map; + +public class UserProfileSearcher extends Searcher { + + public Result search(Query query, Execution execution) { + + Object userIdProperty = query.properties().get("user_id"); + if (userIdProperty != null) { + + // Retrieve user profile... + Tensor userProfile = retrieveUserProfile(userIdProperty.toString(), execution); + + // ... and add user profile to query properties so BlogTensorSearcher can pick it up + query.properties().set(new CompoundName("user_item_cf"), userProfile); + + if (query.isTraceable(9)) { + String tensorRepresentation = userProfile != null ? userProfile.toString() : ""; + query.trace("Setting user profile to :" + tensorRepresentation, 9); + } + } + + return execution.search(query); + } + + + private Tensor retrieveUserProfile(String userId, Execution execution) { + Query query = new Query(); + query.getModel().setRestrict("user"); + query.getModel().getQueryTree().setRoot(new WordItem(userId, "user_id")); + query.setHits(1); + + SearchChain vespaChain = execution.searchChainRegistry().getComponent("vespa"); + Result result = new Execution(vespaChain, execution.context()).search(query); + + // This is needed to get the actual summary data + execution.fill(result); + + Hit hit = getFirstHit(result); + if (hit != null) { + Object userItemCf = hit.getField("user_item_cf"); + if (userItemCf instanceof Inspectable) { + return convertTensor((Inspectable) userItemCf); + } + } + return null; + } + + private Hit getFirstHit(Result result) { + Iterator hiterator = result.hits().deepIterator(); + return hiterator.hasNext() ? hiterator.next() : null; + } + + private Tensor convertTensor(Inspectable field) { + MapTensorBuilder tensorBuilder = new MapTensorBuilder(); + + Inspector cells = field.inspect().field("cells"); + for (Inspector cell : cells.entries()) { + MapTensorBuilder.CellBuilder cellBuilder = tensorBuilder.cell(); + + Inspector address = cell.field("address"); + for (Map.Entry entry : address.fields()) { + String dim = entry.getKey(); + String label = entry.getValue().asString(); + cellBuilder.label(dim, label); + } + + Inspector value = cell.field("value"); + cellBuilder.value(value.asDouble()); + } + return tensorBuilder.build(); + } + +} -- cgit v1.2.3