From 6bfd705b3299c7122ec554388242c0df033a642d Mon Sep 17 00:00:00 2001 From: freva Date: Wed, 21 Dec 2016 13:32:49 +0100 Subject: Move basic-search-for-docker to node-admin root to fix tests --- node-admin/README.md | 2 +- .../basic-search-for-docker/music-data-1.json | 9 ++ .../basic-search-for-docker/music-data-2.json | 9 ++ node-admin/basic-search-for-docker/pom.xml | 85 +++++++++++++++ .../src/main/application/deployment.xml | 5 + .../main/application/searchdefinitions/music.sd | 40 +++++++ .../src/main/application/services.xml | 81 ++++++++++++++ .../yahoo/example/ExampleDocumentProcessor.java | 120 +++++++++++++++++++++ .../java/com/yahoo/example/ExampleSearcher.java | 30 ++++++ .../main/resources/configdefinitions/message.def | 3 + .../src/test/application/services.xml | 20 ++++ .../java/BasicSearchConditionalStagingTest.java | 16 +++ .../src/test/java/BasicSearchStagingTest.java | 27 +++++ .../src/test/java/BasicSearchSystemTest.java | 50 +++++++++ .../com/yahoo/example/ExampleSearcherTest.java | 38 +++++++ .../java/com/yahoo/example/ExampleSystemTest.java | 39 +++++++ .../src/test/resources/minifeed.json | 12 +++ .../resources/sd-change-requiring-restart.patch | 13 +++ .../hosted/node/admin/docker/LocalZoneUtils.java | 5 +- .../basic-search-for-docker/music-data-1.json | 9 -- .../basic-search-for-docker/music-data-2.json | 9 -- .../test/resources/basic-search-for-docker/pom.xml | 85 --------------- .../src/main/application/deployment.xml | 5 - .../main/application/searchdefinitions/music.sd | 40 ------- .../src/main/application/services.xml | 81 -------------- .../yahoo/example/ExampleDocumentProcessor.java | 120 --------------------- .../java/com/yahoo/example/ExampleSearcher.java | 30 ------ .../main/resources/configdefinitions/message.def | 3 - .../src/test/application/services.xml | 20 ---- .../java/BasicSearchConditionalStagingTest.java | 16 --- .../src/test/java/BasicSearchStagingTest.java | 27 ----- .../src/test/java/BasicSearchSystemTest.java | 50 --------- .../com/yahoo/example/ExampleSearcherTest.java | 38 ------- .../java/com/yahoo/example/ExampleSystemTest.java | 39 ------- .../src/test/resources/minifeed.json | 12 --- .../resources/sd-change-requiring-restart.patch | 13 --- 36 files changed, 601 insertions(+), 600 deletions(-) create mode 100644 node-admin/basic-search-for-docker/music-data-1.json create mode 100644 node-admin/basic-search-for-docker/music-data-2.json create mode 100644 node-admin/basic-search-for-docker/pom.xml create mode 100644 node-admin/basic-search-for-docker/src/main/application/deployment.xml create mode 100644 node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd create mode 100644 node-admin/basic-search-for-docker/src/main/application/services.xml create mode 100644 node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java create mode 100644 node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java create mode 100644 node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def create mode 100644 node-admin/basic-search-for-docker/src/test/application/services.xml create mode 100644 node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java create mode 100644 node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java create mode 100644 node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java create mode 100644 node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java create mode 100644 node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java create mode 100644 node-admin/basic-search-for-docker/src/test/resources/minifeed.json create mode 100644 node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/music-data-1.json delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/music-data-2.json delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/pom.xml delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/application/deployment.xml delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/application/searchdefinitions/music.sd delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/application/services.xml delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/main/resources/configdefinitions/message.def delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/application/services.xml delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/resources/minifeed.json delete mode 100644 node-admin/src/test/resources/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch diff --git a/node-admin/README.md b/node-admin/README.md index f76bf46d110..7b3e9805d21 100644 --- a/node-admin/README.md +++ b/node-admin/README.md @@ -45,7 +45,7 @@ To start a local zone, simply run: Package and deploy vespa application by running: ``` - Path pathToApp = Paths.get("node-admin/src/test/resources/basic-search-for-docker"); + Path pathToApp = Paths.get("node-admin/basic-search-for-docker"); runVespaLocal.deployApplication(pathToApp); ``` If the deployment is successful, the final log entry should be something like diff --git a/node-admin/basic-search-for-docker/music-data-1.json b/node-admin/basic-search-for-docker/music-data-1.json new file mode 100644 index 00000000000..44ce547370e --- /dev/null +++ b/node-admin/basic-search-for-docker/music-data-1.json @@ -0,0 +1,9 @@ +{ + "fields": { + "album": "Bad", + "artist": "Michael Jackson", + "title": "Bad", + "year": 1987, + "duration": 247 + } +} diff --git a/node-admin/basic-search-for-docker/music-data-2.json b/node-admin/basic-search-for-docker/music-data-2.json new file mode 100644 index 00000000000..c8e2d3c017c --- /dev/null +++ b/node-admin/basic-search-for-docker/music-data-2.json @@ -0,0 +1,9 @@ +{ + "fields": { + "album": "Recovery", + "artist": "Eminem", + "title": "So Bad", + "year": 2010 + } +} + diff --git a/node-admin/basic-search-for-docker/pom.xml b/node-admin/basic-search-for-docker/pom.xml new file mode 100644 index 00000000000..749640b5a34 --- /dev/null +++ b/node-admin/basic-search-for-docker/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + com.yahoo.example + basic-application + container-plugin + 1.0.1 + application + + + UTF-8 + true + docker + default + + + + com.yahoo.vespa.tenant + base + RELEASE + + + + + ymaven + http://ymaven.corp.yahoo.com:9999/proximity/repository/public + + false + + + + maven2-repository.dev.java.net + http://download.java.net/maven/2 + + false + + + + vespa-maven-release + vespa-maven-release + http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local + + true + + + false + + + + + + ymaven + http://ymaven.corp.yahoo.com:9999/proximity/repository/public + + false + + + + vespa-maven-release + vespa-maven-release + http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local + + true + + + false + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all + -Werror + + + + + + diff --git a/node-admin/basic-search-for-docker/src/main/application/deployment.xml b/node-admin/basic-search-for-docker/src/main/application/deployment.xml new file mode 100644 index 00000000000..0a2397bbca9 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/application/deployment.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd b/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd new file mode 100644 index 00000000000..d6bcaac8ae0 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/application/searchdefinitions/music.sd @@ -0,0 +1,40 @@ +search music { + document music { + field artist type string { + indexing: summary | index + } + field artistId type string { + indexing: summary | attribute + } + + field title type string { + indexing: summary | index + } + + field album type string { + indexing: index + } + + field duration type int { + indexing: summary + } + + field year type int { + indexing: summary | attribute + } + + field popularity type int { + indexing: summary | attribute + } + } + + fieldset default { + fields: artist, title, album + } + + rank-profile song inherits default { + first-phase { + expression:nativeRank(artist,title,album) + if(isNan(attribute(popularity)) == 1, 0,attribute(popularity)) + } + } +} diff --git a/node-admin/basic-search-for-docker/src/main/application/services.xml b/node-admin/basic-search-for-docker/src/main/application/services.xml new file mode 100644 index 00000000000..cea6861ca91 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/application/services.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + Hello, Vespa! (test build iteration 90) + + + + + + + + + + 2 + + + + + + + 1 + + + + diff --git a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java new file mode 100644 index 00000000000..907ab355288 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java @@ -0,0 +1,120 @@ +package com.yahoo.example; + +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentUpdate; + +import java.net.*; +import java.io.*; + +import com.yahoo.document.datatypes.StringFieldValue; +import org.json.*; +import java.util.logging.Logger; + +/** + * A document processor + * + * @author Joe Developer + */ + + +public class ExampleDocumentProcessor extends DocumentProcessor { + + private static final String artistField = "artist"; + private static final String artistIdField = "artistId"; + private static final String spotifyUrl = "https://api.spotify.com/v1/search?type=artist&limit=1&q="; + private static final Logger log = Logger.getLogger(ExampleDocumentProcessor.class.getName()); + + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document document = ((DocumentPut) op).getDocument(); + addArtistId(document); + } else if (op instanceof DocumentUpdate) { + DocumentUpdate update = (DocumentUpdate) op; + //TODO do something to 'update' here + } else if (op instanceof DocumentRemove) { + DocumentRemove remove = (DocumentRemove) op; + //TODO do something to 'remove' here + } + } + return Progress.DONE; + } + + /** + * Query Spotify API, parse JSON and set Artist ID + * + * @param document a Vespa Document + */ + public void addArtistId(Document document) { + StringFieldValue artistString = (StringFieldValue) document.getFieldValue(artistField); + HttpURLConnection conn = null; + try { + String url = spotifyUrl + java.net.URLEncoder.encode(artistString.getString(), "UTF-8"); + conn = getConnection(url); + String artistId = parseSpotifyResponse(new InputStreamReader(conn.getInputStream(), "UTF-8")); + if (artistId == null) { + return; + } + document.setFieldValue(artistIdField, new StringFieldValue(artistId)); + + } catch (Exception e) { + log.warning("Error: " ); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + /** + * @param streamReader the response to read from + * @return artist id from spotify or null if not found + * @throws IOException + */ + + private String parseSpotifyResponse(InputStreamReader streamReader) throws IOException { + // Read JSON data from API + BufferedReader reader = new BufferedReader(streamReader); + StringBuilder builder = new StringBuilder(); + for (String line; (line = reader.readLine()) != null; ) { + builder.append(line).append("\n"); + } + + // Parse the JSON to find the first artist item returned or null if not found + try { + JSONObject json = new JSONObject(builder.toString()); + JSONObject artists = json.getJSONObject("artists"); + JSONArray items = artists.getJSONArray("items"); + JSONObject artist = items.getJSONObject(0); + return artist.getString("id"); + } catch (JSONException e) { + return null; + } + + } + + /** + * Establishes an HTTP Connection + * + * @param inputUrl a String giving the URL to connect to + * @return an HttpURLConnection + * @throws java.io.IOException when connection to inputUrl failed + */ + private HttpURLConnection getConnection(String inputUrl) throws IOException { + URL url = new URL(inputUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + + conn.setRequestProperty("User-Agent", "Vespa Tutorial DocProc"); + conn.setReadTimeout(10000); + conn.setConnectTimeout(5000); + conn.connect(); + + return conn; + } +} + diff --git a/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java new file mode 100644 index 00000000000..cf4c1fc8e34 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java @@ -0,0 +1,30 @@ +package com.yahoo.example; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * A searcher adding a new hit. + * + * @author Joe Developer + */ +public class ExampleSearcher extends Searcher { + public static final String hitId = "ExampleHit"; + private final String message; + + public ExampleSearcher(MessageConfig config) { + message = config.message(); + } + + public Result search(Query query, Execution execution) { + Hit hit = new Hit(hitId); + hit.setField("message", message); + + Result result = execution.search(query); + result.hits().add(hit); + return result; + } +} diff --git a/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def b/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def new file mode 100644 index 00000000000..d2a2bc5407a --- /dev/null +++ b/node-admin/basic-search-for-docker/src/main/resources/configdefinitions/message.def @@ -0,0 +1,3 @@ +version=1 +namespace=example +message string default="Hello, World!" \ No newline at end of file diff --git a/node-admin/basic-search-for-docker/src/test/application/services.xml b/node-admin/basic-search-for-docker/src/test/application/services.xml new file mode 100644 index 00000000000..388ab448f9c --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/application/services.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + Hello, Vespa! + + + + + + + diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java new file mode 100644 index 00000000000..4a6c48073c1 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java @@ -0,0 +1,16 @@ +import com.yahoo.vespa.tenant.systemtest.base.StagingAutoTestConfigBuilder; +import com.yahoo.vespa.tenant.systemtest.base.StagingTest; +import org.junit.Test; + +import java.io.IOException; + +/** + * This test is only run if there is a production instance to get queries and documents from, otherwise the tests + * are marked as ignored automatically. + */ +public class BasicSearchConditionalStagingTest extends StagingTest { + @Test + public void testProdQueriesAndDocuments() throws IOException { + testAutomatically(new StagingAutoTestConfigBuilder().withRoute("music-index").withDocumentType("music").build()); + } +} diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java new file mode 100644 index 00000000000..89bf52a9870 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java @@ -0,0 +1,27 @@ +import com.yahoo.vespa.tenant.systemtest.base.StagingTest; +import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; +import com.yahoo.vespa.tenant.systemtest.blackbox.Report; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class BasicSearchStagingTest extends StagingTest { + @Test + public void testSearchAndFeeding() throws Exception { + + /** + * Generates 100 documents and test recall while feeding. + */ + Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( + " {\n" + + " \"put\": \"id:sampleapp:music::::$1\",\n" + + " \"fields\": {\n" + + " \"title\": \"$2\"\n" + + " }\n" + + " }") + .numberOfDocumentsToFeed(100).build(getStagingTestInstance().getDefaultVespaEndpoint().getUri()) + .testFeedingAndRecall(); + assertThat(report.allSuccess(), is(true)); + } +} diff --git a/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java b/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java new file mode 100644 index 00000000000..e9e72ca4d8c --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java @@ -0,0 +1,50 @@ +import com.yahoo.vespa.tenant.systemtest.Endpoint; +import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; +import com.yahoo.vespa.tenant.systemtest.base.SystemTest; +import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; +import com.yahoo.vespa.tenant.systemtest.blackbox.Report; +import org.junit.Before; +import org.junit.Test; + +import java.net.URI; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Tests running in CI-pipeline. In a regular vespa system this will be triggered whenever a commit is made and + * run before code is pushed to production. + * + * @author dybdahl + */ +public class BasicSearchSystemTest extends SystemTest { + @Override + /** + * In order to develop and test the system tests, you can create a dev instance and push this + * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this + * function to point to this instance. This is only for testing and debugging the test. + */ + protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { + return new VespaEndpoints.Builder().fromPom().inDevCluster() + .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); + } + + + @Test + public void testSearchAndFeeding() throws Exception { + + /** + * Generates 100 documents and test recall while feeding. + */ + Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( + " {\n" + + " \"put\": \"id:sampleapp:music::::$1\",\n" + + " \"fields\": {\n" + + " \"title\": \"$2\"\n" + + " }\n" + + " }") + .numberOfDocumentsToFeed(100).build(getSystemTestsInstance().getDefaultVespaEndpoint().getUri()) + .testFeedingAndRecall(); + assertThat(report.allSuccess(), is(true)); + } +} diff --git a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java new file mode 100644 index 00000000000..87592f2a2ba --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java @@ -0,0 +1,38 @@ +package com.yahoo.example; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import com.yahoo.application.container.Search; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author tonyv + */ +public class ExampleSearcherTest { + @Test + public void hit_is_added() throws Exception { + try (Application app = Application.fromApplicationPackage( + Paths.get("src/test/application"), + Networking.disable)) + { + Search search = app.getJDisc("jdisc").search(); + Result result = search.process(ComponentSpecification.fromString("default"), new Query("?query=ignored")); + + Hit hit = result.hits().get(ExampleSearcher.hitId); + assertNotNull("Hit was not added by ExampleSearcher", hit); + + Object messageFromConfig = "Hello, Vespa!"; + assertThat(hit.getField("message"), is(messageFromConfig)); + } + } +} diff --git a/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java new file mode 100644 index 00000000000..2bed440868e --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java @@ -0,0 +1,39 @@ +package com.yahoo.example; + +import com.yahoo.vespa.tenant.systemtest.Feed; +import com.yahoo.vespa.tenant.systemtest.FeedResult; +import com.yahoo.vespa.tenant.systemtest.Query; +import com.yahoo.vespa.tenant.systemtest.QueryResult; +import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; +import com.yahoo.vespa.tenant.systemtest.base.MutableVespaEndpoint; +import com.yahoo.vespa.tenant.systemtest.base.SystemTest; +import com.yahoo.vespa.tenant.systemtest.hitchecker.HitChecker; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class ExampleSystemTest extends SystemTest { + @Override + /** + * In order to develop and test the system tests, you can create a dev instance and push this + * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this + * function to point to this instance. This is only for testing and debugging the test. + */ + protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { + return new VespaEndpoints.Builder().fromPom().inDevCluster() + .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); + } + @Test + public void testWithOneDocument() throws Exception { + MutableVespaEndpoint endpoint = getSystemTestsInstance().getDefaultVespaEndpoint(); + FeedResult feedResult = endpoint.feed(Feed.createFromResource("/minifeed.json")); + assertThat(feedResult.numOk(), is(1l)); + QueryResult result = endpoint.search(new Query("bad")); + assertThat(result.totalHitCount(), is(1l)); + result.expectHit(1, new HitChecker() + .fieldRegex("title", ".*Bad") + .relevance(0.254, 0.2) + .fieldNull("nosuchfield")); + } +} diff --git a/node-admin/basic-search-for-docker/src/test/resources/minifeed.json b/node-admin/basic-search-for-docker/src/test/resources/minifeed.json new file mode 100644 index 00000000000..2fd5d9e2053 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/resources/minifeed.json @@ -0,0 +1,12 @@ +[ + { + "fields": { + "album": "Bad", + "artist": "Michael Jackson", + "title": "Bad", + "year": 1987 + }, + "put": "id:tutorial:music::1" + } +] + diff --git a/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch b/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch new file mode 100644 index 00000000000..b4e0745fcc2 --- /dev/null +++ b/node-admin/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch @@ -0,0 +1,13 @@ +diff --git a/src/main/application/searchdefinitions/music.sd b/src/main/application/searchdefinitions/music.sd +index d6bcaac..158c217 100644 +--- a/src/main/application/searchdefinitions/music.sd ++++ b/src/main/application/searchdefinitions/music.sd +@@ -16,7 +16,7 @@ search music { + } + + field duration type int { +- indexing: summary ++ indexing: summary | attribute + } + + field year type int { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java index 70df830285c..3d0847bc53c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java @@ -237,11 +237,12 @@ public class LocalZoneUtils { return getContainersForApp(TENANT_NAME, APPLICATION_NAME, "default"); } + @SuppressWarnings("unchecked") public static Set getContainersForApp(String tenant, String application, String instance) { String app = String.join(".", tenant, application, instance); - Map response = requestExecutor.get("/nodes/v2/node/?recursive=true&clusterType=container&application=" + app, + Map response = requestExecutor.get("/nodes/v2/node/?recursive=true&clusterType=container&application=" + app, CONFIG_SERVER_WEB_SERVICE_PORT, Map.class); - List nodes = (List) response.get("nodes"); + List> nodes = (List>) response.get("nodes"); return nodes.stream().map(nodeMap -> (String) nodeMap.get("hostname")).collect(Collectors.toSet()); } diff --git a/node-admin/src/test/resources/basic-search-for-docker/music-data-1.json b/node-admin/src/test/resources/basic-search-for-docker/music-data-1.json deleted file mode 100644 index 44ce547370e..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/music-data-1.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "fields": { - "album": "Bad", - "artist": "Michael Jackson", - "title": "Bad", - "year": 1987, - "duration": 247 - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/music-data-2.json b/node-admin/src/test/resources/basic-search-for-docker/music-data-2.json deleted file mode 100644 index c8e2d3c017c..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/music-data-2.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "fields": { - "album": "Recovery", - "artist": "Eminem", - "title": "So Bad", - "year": 2010 - } -} - diff --git a/node-admin/src/test/resources/basic-search-for-docker/pom.xml b/node-admin/src/test/resources/basic-search-for-docker/pom.xml deleted file mode 100644 index 749640b5a34..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - 4.0.0 - com.yahoo.example - basic-application - container-plugin - 1.0.1 - application - - - UTF-8 - true - docker - default - - - - com.yahoo.vespa.tenant - base - RELEASE - - - - - ymaven - http://ymaven.corp.yahoo.com:9999/proximity/repository/public - - false - - - - maven2-repository.dev.java.net - http://download.java.net/maven/2 - - false - - - - vespa-maven-release - vespa-maven-release - http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local - - true - - - false - - - - - - ymaven - http://ymaven.corp.yahoo.com:9999/proximity/repository/public - - false - - - - vespa-maven-release - vespa-maven-release - http://edge.artifactory.yahoo.com:8000/artifactory/vespa-maven-libs-release-local - - true - - - false - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - -Xlint:all - -Werror - - - - - - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/deployment.xml b/node-admin/src/test/resources/basic-search-for-docker/src/main/application/deployment.xml deleted file mode 100644 index 0a2397bbca9..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/deployment.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/searchdefinitions/music.sd b/node-admin/src/test/resources/basic-search-for-docker/src/main/application/searchdefinitions/music.sd deleted file mode 100644 index d6bcaac8ae0..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/searchdefinitions/music.sd +++ /dev/null @@ -1,40 +0,0 @@ -search music { - document music { - field artist type string { - indexing: summary | index - } - field artistId type string { - indexing: summary | attribute - } - - field title type string { - indexing: summary | index - } - - field album type string { - indexing: index - } - - field duration type int { - indexing: summary - } - - field year type int { - indexing: summary | attribute - } - - field popularity type int { - indexing: summary | attribute - } - } - - fieldset default { - fields: artist, title, album - } - - rank-profile song inherits default { - first-phase { - expression:nativeRank(artist,title,album) + if(isNan(attribute(popularity)) == 1, 0,attribute(popularity)) - } - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/services.xml b/node-admin/src/test/resources/basic-search-for-docker/src/main/application/services.xml deleted file mode 100644 index cea6861ca91..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/application/services.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - - - - Hello, Vespa! (test build iteration 90) - - - - - - - - - - 2 - - - - - - - 1 - - - - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java b/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java deleted file mode 100644 index 907ab355288..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleDocumentProcessor.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentUpdate; - -import java.net.*; -import java.io.*; - -import com.yahoo.document.datatypes.StringFieldValue; -import org.json.*; -import java.util.logging.Logger; - -/** - * A document processor - * - * @author Joe Developer - */ - - -public class ExampleDocumentProcessor extends DocumentProcessor { - - private static final String artistField = "artist"; - private static final String artistIdField = "artistId"; - private static final String spotifyUrl = "https://api.spotify.com/v1/search?type=artist&limit=1&q="; - private static final Logger log = Logger.getLogger(ExampleDocumentProcessor.class.getName()); - - public Progress process(Processing processing) { - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - Document document = ((DocumentPut) op).getDocument(); - addArtistId(document); - } else if (op instanceof DocumentUpdate) { - DocumentUpdate update = (DocumentUpdate) op; - //TODO do something to 'update' here - } else if (op instanceof DocumentRemove) { - DocumentRemove remove = (DocumentRemove) op; - //TODO do something to 'remove' here - } - } - return Progress.DONE; - } - - /** - * Query Spotify API, parse JSON and set Artist ID - * - * @param document a Vespa Document - */ - public void addArtistId(Document document) { - StringFieldValue artistString = (StringFieldValue) document.getFieldValue(artistField); - HttpURLConnection conn = null; - try { - String url = spotifyUrl + java.net.URLEncoder.encode(artistString.getString(), "UTF-8"); - conn = getConnection(url); - String artistId = parseSpotifyResponse(new InputStreamReader(conn.getInputStream(), "UTF-8")); - if (artistId == null) { - return; - } - document.setFieldValue(artistIdField, new StringFieldValue(artistId)); - - } catch (Exception e) { - log.warning("Error: " ); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - } - - /** - * @param streamReader the response to read from - * @return artist id from spotify or null if not found - * @throws IOException - */ - - private String parseSpotifyResponse(InputStreamReader streamReader) throws IOException { - // Read JSON data from API - BufferedReader reader = new BufferedReader(streamReader); - StringBuilder builder = new StringBuilder(); - for (String line; (line = reader.readLine()) != null; ) { - builder.append(line).append("\n"); - } - - // Parse the JSON to find the first artist item returned or null if not found - try { - JSONObject json = new JSONObject(builder.toString()); - JSONObject artists = json.getJSONObject("artists"); - JSONArray items = artists.getJSONArray("items"); - JSONObject artist = items.getJSONObject(0); - return artist.getString("id"); - } catch (JSONException e) { - return null; - } - - } - - /** - * Establishes an HTTP Connection - * - * @param inputUrl a String giving the URL to connect to - * @return an HttpURLConnection - * @throws java.io.IOException when connection to inputUrl failed - */ - private HttpURLConnection getConnection(String inputUrl) throws IOException { - URL url = new URL(inputUrl); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - - conn.setRequestProperty("User-Agent", "Vespa Tutorial DocProc"); - conn.setReadTimeout(10000); - conn.setConnectTimeout(5000); - conn.connect(); - - return conn; - } -} - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java b/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java deleted file mode 100644 index cf4c1fc8e34..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/java/com/yahoo/example/ExampleSearcher.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; - -/** - * A searcher adding a new hit. - * - * @author Joe Developer - */ -public class ExampleSearcher extends Searcher { - public static final String hitId = "ExampleHit"; - private final String message; - - public ExampleSearcher(MessageConfig config) { - message = config.message(); - } - - public Result search(Query query, Execution execution) { - Hit hit = new Hit(hitId); - hit.setField("message", message); - - Result result = execution.search(query); - result.hits().add(hit); - return result; - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/main/resources/configdefinitions/message.def b/node-admin/src/test/resources/basic-search-for-docker/src/main/resources/configdefinitions/message.def deleted file mode 100644 index d2a2bc5407a..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/main/resources/configdefinitions/message.def +++ /dev/null @@ -1,3 +0,0 @@ -version=1 -namespace=example -message string default="Hello, World!" \ No newline at end of file diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/application/services.xml b/node-admin/src/test/resources/basic-search-for-docker/src/test/application/services.xml deleted file mode 100644 index 388ab448f9c..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/application/services.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - Hello, Vespa! - - - - - - - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java b/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java deleted file mode 100644 index 4a6c48073c1..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchConditionalStagingTest.java +++ /dev/null @@ -1,16 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.base.StagingAutoTestConfigBuilder; -import com.yahoo.vespa.tenant.systemtest.base.StagingTest; -import org.junit.Test; - -import java.io.IOException; - -/** - * This test is only run if there is a production instance to get queries and documents from, otherwise the tests - * are marked as ignored automatically. - */ -public class BasicSearchConditionalStagingTest extends StagingTest { - @Test - public void testProdQueriesAndDocuments() throws IOException { - testAutomatically(new StagingAutoTestConfigBuilder().withRoute("music-index").withDocumentType("music").build()); - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java b/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java deleted file mode 100644 index 89bf52a9870..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchStagingTest.java +++ /dev/null @@ -1,27 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.base.StagingTest; -import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; -import com.yahoo.vespa.tenant.systemtest.blackbox.Report; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -public class BasicSearchStagingTest extends StagingTest { - @Test - public void testSearchAndFeeding() throws Exception { - - /** - * Generates 100 documents and test recall while feeding. - */ - Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( - " {\n" + - " \"put\": \"id:sampleapp:music::::$1\",\n" + - " \"fields\": {\n" + - " \"title\": \"$2\"\n" + - " }\n" + - " }") - .numberOfDocumentsToFeed(100).build(getStagingTestInstance().getDefaultVespaEndpoint().getUri()) - .testFeedingAndRecall(); - assertThat(report.allSuccess(), is(true)); - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java b/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java deleted file mode 100644 index e9e72ca4d8c..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/BasicSearchSystemTest.java +++ /dev/null @@ -1,50 +0,0 @@ -import com.yahoo.vespa.tenant.systemtest.Endpoint; -import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; -import com.yahoo.vespa.tenant.systemtest.base.SystemTest; -import com.yahoo.vespa.tenant.systemtest.blackbox.BlackBoxTester; -import com.yahoo.vespa.tenant.systemtest.blackbox.Report; -import org.junit.Before; -import org.junit.Test; - -import java.net.URI; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * Tests running in CI-pipeline. In a regular vespa system this will be triggered whenever a commit is made and - * run before code is pushed to production. - * - * @author dybdahl - */ -public class BasicSearchSystemTest extends SystemTest { - @Override - /** - * In order to develop and test the system tests, you can create a dev instance and push this - * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this - * function to point to this instance. This is only for testing and debugging the test. - */ - protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { - return new VespaEndpoints.Builder().fromPom().inDevCluster() - .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); - } - - - @Test - public void testSearchAndFeeding() throws Exception { - - /** - * Generates 100 documents and test recall while feeding. - */ - Report report = new BlackBoxTester.BlackBoxTesterBuilder().documentFormat( - " {\n" + - " \"put\": \"id:sampleapp:music::::$1\",\n" + - " \"fields\": {\n" + - " \"title\": \"$2\"\n" + - " }\n" + - " }") - .numberOfDocumentsToFeed(100).build(getSystemTestsInstance().getDefaultVespaEndpoint().getUri()) - .testFeedingAndRecall(); - assertThat(report.allSuccess(), is(true)); - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java b/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java deleted file mode 100644 index 87592f2a2ba..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSearcherTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.application.Application; -import com.yahoo.application.Networking; -import com.yahoo.application.container.Search; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author tonyv - */ -public class ExampleSearcherTest { - @Test - public void hit_is_added() throws Exception { - try (Application app = Application.fromApplicationPackage( - Paths.get("src/test/application"), - Networking.disable)) - { - Search search = app.getJDisc("jdisc").search(); - Result result = search.process(ComponentSpecification.fromString("default"), new Query("?query=ignored")); - - Hit hit = result.hits().get(ExampleSearcher.hitId); - assertNotNull("Hit was not added by ExampleSearcher", hit); - - Object messageFromConfig = "Hello, Vespa!"; - assertThat(hit.getField("message"), is(messageFromConfig)); - } - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java b/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java deleted file mode 100644 index 2bed440868e..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/java/com/yahoo/example/ExampleSystemTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.yahoo.example; - -import com.yahoo.vespa.tenant.systemtest.Feed; -import com.yahoo.vespa.tenant.systemtest.FeedResult; -import com.yahoo.vespa.tenant.systemtest.Query; -import com.yahoo.vespa.tenant.systemtest.QueryResult; -import com.yahoo.vespa.tenant.systemtest.VespaEndpoints; -import com.yahoo.vespa.tenant.systemtest.base.MutableVespaEndpoint; -import com.yahoo.vespa.tenant.systemtest.base.SystemTest; -import com.yahoo.vespa.tenant.systemtest.hitchecker.HitChecker; -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class ExampleSystemTest extends SystemTest { - @Override - /** - * In order to develop and test the system tests, you can create a dev instance and push this - * (e.g. mvn deploy:vespa) to your personal vespa dev instance. You will need to change this - * function to point to this instance. This is only for testing and debugging the test. - */ - protected VespaEndpoints createVespaSystemTestInstanceEndpointsWhenNotOnScrewdriver() { - return new VespaEndpoints.Builder().fromPom().inDevCluster() - .withRegion("corp-us-east-1").withTenant("ENTER TENANT USER FOR USED FOR LOCAL DEVELOPMENT HERE").build(); - } - @Test - public void testWithOneDocument() throws Exception { - MutableVespaEndpoint endpoint = getSystemTestsInstance().getDefaultVespaEndpoint(); - FeedResult feedResult = endpoint.feed(Feed.createFromResource("/minifeed.json")); - assertThat(feedResult.numOk(), is(1l)); - QueryResult result = endpoint.search(new Query("bad")); - assertThat(result.totalHitCount(), is(1l)); - result.expectHit(1, new HitChecker() - .fieldRegex("title", ".*Bad") - .relevance(0.254, 0.2) - .fieldNull("nosuchfield")); - } -} diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/minifeed.json b/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/minifeed.json deleted file mode 100644 index 2fd5d9e2053..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/minifeed.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "fields": { - "album": "Bad", - "artist": "Michael Jackson", - "title": "Bad", - "year": 1987 - }, - "put": "id:tutorial:music::1" - } -] - diff --git a/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch b/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch deleted file mode 100644 index b4e0745fcc2..00000000000 --- a/node-admin/src/test/resources/basic-search-for-docker/src/test/resources/sd-change-requiring-restart.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/main/application/searchdefinitions/music.sd b/src/main/application/searchdefinitions/music.sd -index d6bcaac..158c217 100644 ---- a/src/main/application/searchdefinitions/music.sd -+++ b/src/main/application/searchdefinitions/music.sd -@@ -16,7 +16,7 @@ search music { - } - - field duration type int { -- indexing: summary -+ indexing: summary | attribute - } - - field year type int { -- cgit v1.2.3