diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config-model/src/test/java/com |
Publish
Diffstat (limited to 'config-model/src/test/java/com')
269 files changed, 26486 insertions, 0 deletions
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java new file mode 100644 index 00000000000..8958300538b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -0,0 +1,379 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.google.common.io.Files; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.document.DataType; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.UnproperSearch; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.SearchDefinition; +import org.json.JSONException; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.regex.Pattern; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ApplicationDeployTest { + + private static final String TESTDIR = "src/test/cfg/application/"; + private static final String TESTSDDIR = TESTDIR + "app1/searchdefinitions/"; + + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Test + public void testVespaModel() throws SAXException, IOException { + FilesApplicationPackage app = createAppPkg(TESTDIR + "app1"); + assertThat(app.getApplicationName(), is("app1")); + VespaModel model = new VespaModel(app); + List<SearchDefinition> searchDefinitions = getSearchDefinitions(app); + assertEquals(searchDefinitions.size(), 5); + for (SearchDefinition searchDefinition : searchDefinitions) { + Search s = searchDefinition.getSearch(); + switch (s.getName()) { + case "music": + case "laptop": + case "pc": + case "sock": + break; + case "product": + assertTrue(s instanceof UnproperSearch); + assertEquals(s.getDocument().getField("title").getDataType(), DataType.STRING); + break; + default: + fail(); + } + } + File[] truth = new File[]{new File(TESTSDDIR + "laptop.sd"), + new File(TESTSDDIR + "music.sd"), + new File(TESTSDDIR + "pc.sd"), + new File(TESTSDDIR + "product.sd"), + new File(TESTSDDIR + "sock.sd")}; + Arrays.sort(truth); + List<File> appSdFiles = app.getSearchDefinitionFiles(); + Collections.sort(appSdFiles); + assertEquals(appSdFiles, Arrays.asList(truth)); + + List<FilesApplicationPackage.Component> components = app.getComponents(); + assertEquals(1, components.size()); + Map<String, Bundle.DefEntry> defEntries = + defEntries2map(components.get(0).getDefEntries()); + assertEquals(2, defEntries.size()); + System.out.println(defEntries); + Bundle.DefEntry def1 = defEntries.get("test1"); + Bundle.DefEntry def2 = defEntries.get("test2"); + assertNotNull(def1); + assertNotNull(def2); + assertEquals("namespace=config\nintVal int default=0", def1.contents); + assertEquals("namespace=a.b\n\ndoubleVal double default=0.0", def2.contents); + + // Check that getFilename works + ArrayList<String> sdFileNames = new ArrayList<>(); + for (SearchDefinition sd : searchDefinitions) { + sdFileNames.add(sd.getFilename()); + } + Collections.sort(sdFileNames); + assertThat(sdFileNames.get(0), is("laptop.sd")); + assertThat(sdFileNames.get(1), is("music.sd")); + assertThat(sdFileNames.get(2), is("pc.sd")); + assertThat(sdFileNames.get(3), is("product.sd")); + assertThat(sdFileNames.get(4), is("sock.sd")); + } + + @Test + public void testGetFile() throws IOException { + FilesApplicationPackage app = createAppPkg(TESTDIR + "app1"); + try (Reader foo = app.getFile(Path.fromString("files/foo.json")).createReader()) { + assertEquals(IOUtils.readAll(foo), "foo : foo\n"); + } + try (Reader bar = app.getFile(Path.fromString("files/sub/bar.json")).createReader()) { + assertEquals(IOUtils.readAll(bar), "bar : bar\n"); + } + assertTrue(app.getFile(Path.createRoot()).exists()); + assertTrue(app.getFile(Path.createRoot()).isDirectory()); + } + + /* + * Put a list of def entries to a map, with the name as key. This is done because the order + * of the def entries in the list cannot be guaranteed. + */ + private Map<String, Bundle.DefEntry> defEntries2map + (List<Bundle.DefEntry> defEntries) { + Map<String, Bundle.DefEntry> ret = + new HashMap<>(); + + for (Bundle.DefEntry def : defEntries) + ret.put(def.defName, def); + return ret; + } + + @Test + public void testSdFromDocprocBundle() throws IOException, SAXException { + String appDir = "src/test/cfg/application/app_sdbundles"; + FilesApplicationPackage app = createAppPkg(appDir); + VespaModel model = new VespaModel(app); + // Check that the resulting documentmanager config contains those types + DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); + model.getConfig(b, VespaModel.ROOT_CONFIGID); + //String docMan = model.getConfig("documentmanager", "").toString(); + DocumentmanagerConfig dc = new DocumentmanagerConfig(b); + String docMan=ConfigInstance.serialize(dc).toString(); + int pFlags = Pattern.MULTILINE + Pattern.DOTALL; + Pattern base = Pattern.compile(".*name.*base\\.header.*", pFlags); + Pattern book = Pattern.compile(".*name.*book\\.header.*", pFlags); + Pattern music = Pattern.compile(".*name.*music\\.header.*", pFlags); + Pattern video = Pattern.compile(".*name.*video\\.header.*", pFlags); + Pattern muzak = Pattern.compile(".*name.*muzak\\.header.*", pFlags); + assertTrue(base.matcher(docMan).matches()); + assertTrue(book.matcher(docMan).matches()); + assertTrue(music.matcher(docMan).matches()); + assertTrue(video.matcher(docMan).matches()); + assertTrue(muzak.matcher(docMan).matches()); + } + + @Test + public void include_dirs_are_included() throws Exception { + FilesApplicationPackage app = createAppPkg(TESTDIR + "include_dirs"); + + List<String> includeDirs = app.getUserIncludeDirs(); + assertThat(includeDirs, contains("jdisc_dir", "dir1", "dir2", "empty_dir")); + } + + @Test + public void non_existent_include_dir_is_not_allowed() throws Exception { + File appDir = tmpFolder.newFolder("non-existent-include"); + String services = "<services version='1.0'>" + + "<include dir='non-existent' />" + + "</services>\n"; + + IOUtils.writeFile(new File(appDir, "services.xml"), services, false); + try { + FilesApplicationPackage.fromFile(appDir); + fail("Expected exception due to non-existent include dir"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Cannot include directory 'non-existent', as it does not exist")); + } + } + + @Test + public void testThatModelIsRebuiltWhenSearchDefinitionIsAdded() throws IOException { + File tmpDir = Files.createTempDir(); + IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir); + FilesApplicationPackage app = createAppPkg(tmpDir.getAbsolutePath()); + assertThat(getSearchDefinitions(app).size(), is(5)); + File sdDir = new File(tmpDir, "searchdefinitions"); + File sd = new File(sdDir, "testfoo.sd"); + IOUtils.writeFile(sd, "search testfoo { document testfoo { field bar type string { } } }", false); + assertThat(getSearchDefinitions(app).size(), is(6)); + } + + private List<SearchDefinition> getSearchDefinitions(FilesApplicationPackage app) { + return new DeployState.Builder().applicationPackage(app).build().getSearchDefinitions(); + } + + public FilesApplicationPackage createAppPkg(String appPkg) throws IOException { + return createAppPkg(appPkg, true); + } + + public FilesApplicationPackage createAppPkgDoNotValidateXml(String appPkg) throws IOException { + return createAppPkg(appPkg, false); + } + + public FilesApplicationPackage createAppPkg(String appPkg, boolean validateXml) throws IOException { + final FilesApplicationPackage filesApplicationPackage = FilesApplicationPackage.fromFile(new File(appPkg)); + if (validateXml) { + ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createTestXmlValidator(new File(appPkg)); + validator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(filesApplicationPackage); + } + return filesApplicationPackage; + } + + @Test + public void testThatNewServicesFileNameWorks() throws IOException { + String appPkg = TESTDIR + "newfilenames"; + assertEquals(appPkg + "/services.xml", createAppPkgDoNotValidateXml(appPkg).getServicesSource()); + } + + @Test + public void testThatNewHostsFileNameWorks() throws IOException { + String appPkg = TESTDIR + "newfilenames"; + assertEquals(appPkg + "/hosts.xml", createAppPkgDoNotValidateXml(appPkg).getHostSource()); + } + + @Test + public void testGetJars() throws IOException { + String jarName = "src/test/cfg/application/app_sdbundles/components/testbundle.jar"; + JarFile jar = new JarFile(jarName); + Map<String, String> payloads = ApplicationPackage.getBundleSdFiles("", jar); + assertEquals(payloads.size(), 4); + assertTrue(payloads.get("base.sd").startsWith("search base")); + assertTrue(payloads.get("book.sd").startsWith("search book")); + assertTrue(payloads.get("music.sd").startsWith("search music")); + assertTrue(payloads.get("video.sd").startsWith("search video")); + assertTrue(payloads.get("base.sd").endsWith("}")); + assertTrue(payloads.get("book.sd").endsWith("}\n")); + assertTrue(payloads.get("music.sd").endsWith("}\n")); + assertTrue(payloads.get("video.sd").endsWith("}\n")); + } + + @Test + public void testConfigDefinitionsFromJars() throws IOException { + String appName = "src/test/cfg//application/app1"; + FilesApplicationPackage app = FilesApplicationPackage.fromFile(new File(appName)); + Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs = app.getAllExistingConfigDefs(); + assertThat(defs.size(), is(2)); + } + + @Test + public void testMetaData() throws IOException, JSONException { + File tmp = Files.createTempDir(); + String appPkg = TESTDIR + "app1"; + IOUtils.copyDirectory(new File(appPkg), tmp); + final DeployData deployData = new DeployData("foo", "bar", "baz", 13l, 1337l, 3l); + FilesApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); + app.writeMetaData(); + FilesApplicationPackage newApp = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); + ApplicationMetaData meta = newApp.getMetaData(); + assertThat(meta.getDeployedByUser(), is("foo")); + assertThat(meta.getDeployPath(), is("bar")); + assertThat(meta.getDeployTimestamp(), is(13l)); + assertThat(meta.getGeneration(), is(1337l)); + assertThat(meta.getPreviousActiveGeneration(), is(3l)); + final String checkSum = meta.getCheckSum(); + assertNotNull(checkSum); + + assertTrue((new File(tmp, "hosts.xml")).delete()); + FilesApplicationPackage app2 = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); + final String app2CheckSum = app2.getMetaData().getCheckSum(); + assertThat(app2CheckSum, is(not(checkSum))); + + assertTrue((new File(tmp, "files/foo.json")).delete()); + FilesApplicationPackage app3 = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); + final String app3CheckSum = app3.getMetaData().getCheckSum(); + assertThat(app3CheckSum, is(not(app2CheckSum))); + } + + @Test + public void testGetJarEntryName() { + JarEntry e = new JarEntry("/searchdefinitions/foo.sd"); + assertEquals(ApplicationPackage.getFileName(e), "foo.sd"); + e = new JarEntry("bar"); + assertEquals(ApplicationPackage.getFileName(e), "bar"); + e = new JarEntry(""); + assertEquals(ApplicationPackage.getFileName(e), ""); + } + + @After + public void cleanDirs() { + IOUtils.recursiveDeleteDir(new File(TESTDIR + "app1/myDir")); + IOUtils.recursiveDeleteDir(new File(TESTDIR + "app1/searchdefinitions/myDir2")); + IOUtils.recursiveDeleteDir(new File(TESTDIR + "app1/myDir3")); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @After + public void cleanFiles() { + new File(new File(TESTDIR + "app1"),"foo.txt").delete(); + new File(new File(TESTDIR + "app1"),"searchdefinitions/bar.text").delete(); + IOUtils.recursiveDeleteDir(new File(TESTDIR + "app1/mySubDir")); + } + + /** + * Tests that an invalid jar is identified as not being a jar file + */ + @Test + public void testInvalidJar() { + try { + FilesApplicationPackage.getComponents(new File("src/test/cfg/application/validation/invalidjar_app")); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Error opening jar file 'invalid.jar'. Please check that this is a valid jar file")); + } + } + + /** + * Tests that config definitions with namespace are treated properly when they have the format + * as in the config definitions dir ($VESPA_HOME/var/db/vespa/config_server/serverdb/classes on a machine + * with Vespa packages installed) (does not test when read from user def files). Also tests a config + * definition without version in file name + */ + @Test + public void testConfigDefinitionsAndNamespaces() { + final File appDir = new File("src/test/cfg/application/configdeftest"); + FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); + + DeployState deployState = new DeployState.Builder().applicationPackage(app).build(); + + ConfigDefinition def = deployState.getConfigDefinition(new ConfigDefinitionKey("foo", CNode.DEFAULT_NAMESPACE)); + assertThat(def.getNamespace(), is(CNode.DEFAULT_NAMESPACE)); + + def = deployState.getConfigDefinition(new ConfigDefinitionKey("baz", CNode.DEFAULT_NAMESPACE)); + assertThat(def.getNamespace(), is("xyzzy")); + + def = deployState.getConfigDefinition(new ConfigDefinitionKey("foo", "qux")); + assertThat(def.getNamespace(), is("qux")); + + // A config def without version in filename and version in file header + def = deployState.getConfigDefinition(new ConfigDefinitionKey("xyzzy", CNode.DEFAULT_NAMESPACE)); + assertThat(def.getNamespace(), is(CNode.DEFAULT_NAMESPACE)); + assertThat(def.getName(), is("xyzzy")); + + // Without giving namespace, namespace is really CNode.DEFAULT_NAMESPACE + def = deployState.getConfigDefinition(new ConfigDefinitionKey("baz", "")); + assertThat(def.getNamespace(), is("xyzzy")); + + // Without giving namespace, namespace is really xyzzy + def = deployState.getConfigDefinition(new ConfigDefinitionKey("baz", "")); + assertThat(def.getNamespace(), is("xyzzy")); + + // Two defs, one with and one without namespace. The one with namespace should have precedence. + def = deployState.getConfigDefinition(new ConfigDefinitionKey("bar", "xyzzy")); + assertThat(def.getNamespace(), is("xyzzy")); + assertTrue(def.getIntDefs().containsKey("foo")); // xyzzy.baz.def has precedence before baz.def, so foo exists + assertThat(def.getIntDefs().get("bar").getDefVal(), is(2)); + } + + @Test(expected=IllegalArgumentException.class) + public void testDifferentNameOfSdFileAndSearchName() throws SAXException, IOException { + FilesApplicationPackage app = createAppPkg(TESTDIR + "sdfilenametest"); + new DeployState.Builder().applicationPackage(app).build(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/ConfigModelBuilderTest.java b/config-model/src/test/java/com/yahoo/config/model/ConfigModelBuilderTest.java new file mode 100644 index 00000000000..10efd62479d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/ConfigModelBuilderTest.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelBuilderTest { + @Test + public void testEquals() { + ConfigModelBuilder<?> ba = new A.Builder(); + ConfigModelBuilder<?> ba2 = new A2.Builder(); + ConfigModelBuilder<?> bb = new B.Builder(); + ConfigModelBuilder <?>bb2 = new B2.Builder(); + + assertTrue(ba.equals(ba)); + assertTrue(ba.equals(ba2)); + assertFalse(ba.equals(bb)); + assertFalse(ba.equals(bb2)); + + assertTrue(ba2.equals(ba)); + assertTrue(ba2.equals(ba2)); + assertFalse(ba2.equals(bb)); + assertFalse(ba2.equals(bb2)); + + assertFalse(bb.equals(ba)); + assertFalse(bb.equals(ba2)); + assertTrue(bb.equals(bb)); + assertFalse(bb.equals(bb2)); + + assertFalse(bb2.equals(ba)); + assertFalse(bb2.equals(ba2)); + assertFalse(bb2.equals(bb)); + assertTrue(bb2.equals(bb2)); + + assertFalse(ba.equals(new ArrayList<>())); + } + + private static class A extends ConfigModel { + public A(ConfigModelContext modelContext) { super(modelContext); } + public static class Builder extends ConfigModelBuilder<A> { + public Builder() { super(A.class); } + + @Override + public List<ConfigModelId> handlesElements() { + List<ConfigModelId> ids = new ArrayList<>(); + ids.add(ConfigModelId.fromName("foo")); + ids.add(ConfigModelId.fromName("bar")); + return ids; + } + + @Override + public void doBuild(A model, Element spec, ConfigModelContext modelContext) { } + } + + } + + private static class A2 extends ConfigModel { + public A2(ConfigModelContext modelContext) { super(modelContext); } + public static class Builder extends ConfigModelBuilder<A2> { + public Builder() { super(A2.class); } + + @Override + public List<ConfigModelId> handlesElements() { + List<ConfigModelId> ids = new ArrayList<>(); + ids.add(ConfigModelId.fromName("foo")); + ids.add(ConfigModelId.fromName("bar")); + return ids; + } + + @Override + public void doBuild(A2 model, Element spec, ConfigModelContext modelContext) { } + } + } + + private static class B extends ConfigModel { + public B(ConfigModelContext modelContext) { super(modelContext); } + public static class Builder extends ConfigModelBuilder<B> { + public Builder() { super(B.class); } + + @Override + public List<ConfigModelId> handlesElements() { + List<ConfigModelId> ids = new ArrayList<>(); + ids.add(ConfigModelId.fromName("bar")); + return ids; + } + + @Override + public void doBuild(B model, Element spec, ConfigModelContext modelContext) { } + } + } + + private static class B2 extends ConfigModel { + public B2(ConfigModelContext modelContext) { super(modelContext); } + public static class Builder extends ConfigModelBuilder<B2> { + public Builder() { super(B2.class); } + + @Override + public List<ConfigModelId> handlesElements() { + List<ConfigModelId> ids = new ArrayList<>(); + ids.add(ConfigModelId.fromName("foo")); + ids.add(ConfigModelId.fromName("bim")); + return ids; + } + + @Override + public void doBuild(B2 model, Element spec, ConfigModelContext modelContext) { } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/ConfigModelContextTest.java b/config-model/src/test/java/com/yahoo/config/model/ConfigModelContextTest.java new file mode 100644 index 00000000000..4deb8d427c4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/ConfigModelContextTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelContextTest { + @Test + public void testConfigModelContext() { + AbstractConfigProducer root = new MockRoot(); + String id = "foobar"; + ApplicationPackage pkg = new MockApplicationPackage.Builder() + .withServices("<services version=\"1.0\"><admin version=\"2.0\" /></services>") + .build(); + DeployState deployState = DeployState.createTestState(pkg); + DeployLogger logger = deployState.getDeployLogger(); + ConfigModelContext ctx = ConfigModelContext.create(deployState, null, root, id); + assertThat(ctx.getApplicationPackage(), is(pkg)); + assertThat(ctx.getProducerId(), is(id)); + assertThat(ctx.getParentProducer(), is(root)); + assertThat(ctx.getDeployLogger(), is(logger)); + ctx = ConfigModelContext.createFromParentAndId(null, root, id); + assertThat(ctx.getProducerId(), is(id)); + assertThat(ctx.getParentProducer(), is(root)); + AbstractConfigProducer newRoot = new MockRoot("bar"); + ctx = ctx.modifyParent(newRoot); + assertThat(ctx.getProducerId(), is(id)); + assertThat(ctx.getParentProducer(), is(not(root))); + assertThat(ctx.getParentProducer(), is(newRoot)); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/ConfigModelUtilsTest.java b/config-model/src/test/java/com/yahoo/config/model/ConfigModelUtilsTest.java new file mode 100644 index 00000000000..1630c317825 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/ConfigModelUtilsTest.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.application.provider.Bundle; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +import static junit.framework.TestCase.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelUtilsTest { + + /** + * Tests that a def file both with and without namespace in file name are handled, and that + * def files in other directories than 'configdefinitions/' within the jar file are ignored. + */ + @Test + public void testDefFilesInBundle() { + List<Bundle> bundles = Bundle.getBundles(new File("src/test/cfg/application/app1/components/")); + assertThat(bundles.size(), is(1)); + Bundle bundle = bundles.get(0); + assertThat(bundle.getDefEntries().size(), is(2)); + + Bundle.DefEntry defEntry1 = bundle.getDefEntries().get(0); + Bundle.DefEntry defEntry2; + List<Bundle.DefEntry> defEntries = bundle.getDefEntries(); + if (defEntry1.defName.equals("test1")) { + defEntry2 = defEntries.get(1); + } else { + defEntry1 = defEntries.get(1); + defEntry2 = defEntries.get(0); + } + assertThat(defEntry1.defName, is("test1")); + assertThat(defEntry1.defNamespace, is("config")); + + assertThat(defEntry2.defName, is("test2")); + assertThat(defEntry2.defNamespace, is("a.b")); + } + + /** + * Tests that an invalid jar is identified as not being a jar file + */ + @Test + public void testInvalidJar() { + try { + Bundle.getBundles(new File("src/test/cfg/application/validation/invalidjar_app/components")); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Error opening jar file 'invalid.jar'. Please check that this is a valid jar file")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java b/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java new file mode 100644 index 00000000000..ea32feff8a9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class MapConfigModelRegistryTest { + + @Test + public void require_that_registry_finds_components() { + ModelABuilder ba = new ModelABuilder(); + ModelBBuilder bb = new ModelBBuilder(); + ConfigModelRegistry registry = MapConfigModelRegistry.createFromList(ba, bb); + assertNotNull(registry.resolve(ConfigModelId.fromName("modelA"))); + assertNotNull(registry.resolve(ConfigModelId.fromName("modelB"))); + assertEquals(ba, registry.resolve(ConfigModelId.fromName("modelA")).iterator().next()); + assertEquals(bb, registry.resolve(ConfigModelId.fromName("modelB")).iterator().next()); + assertTrue(registry.resolve(ConfigModelId.fromName("modelC")).isEmpty()); + } + + @Test + public void require_all_builders_for_a_tag() { + ModelBBuilder b1 = new ModelBBuilder(); + ModelB2Builder b2 = new ModelB2Builder(); + ConfigModelRegistry registry = MapConfigModelRegistry.createFromList(b1, b2); + Collection<ConfigModelBuilder> builders = registry.resolve(ConfigModelId.fromName("modelB")); + assertEquals(2, builders.size()); + assertEquals(b1, builders.iterator().next()); + assertEquals(b2, builders.iterator().next()); + } + + private static class ModelB2Builder extends ConfigModelBuilder<ModelB> { + public ModelB2Builder() { + super(ModelB.class); + } + + @Override + public List<ConfigModelId> handlesElements() { return Arrays.asList(ConfigModelId.fromName("modelB")); } + @Override + public void doBuild(ModelB model, Element spec, ConfigModelContext modelContext) { } + } + + private static class ModelBBuilder extends ConfigModelBuilder<ModelB> { + public ModelBBuilder() { + super(ModelB.class); + } + @Override + public List<ConfigModelId> handlesElements() { return Arrays.asList(ConfigModelId.fromName("modelB")); } + @Override + public void doBuild(ModelB model, Element spec, ConfigModelContext modelContext) { } + } + + private class ModelB extends ConfigModel { + protected ModelB(ConfigModelContext modelContext) { + super(modelContext); + } + } + + private static class ModelABuilder extends ConfigModelBuilder<ModelA> { + public ModelABuilder() { + super(ModelA.class); + } + @Override + public List<ConfigModelId> handlesElements() { return Arrays.asList(ConfigModelId.fromName("modelA")); } + + @Override + public void doBuild(ModelA model, Element spec, ConfigModelContext modelContext) { } + } + + private class ModelA extends ConfigModel { + protected ModelA(ConfigModelContext modelContext) { + super(modelContext); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java new file mode 100644 index 00000000000..c462cde3fb2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.api.*; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** +* @author musum +*/ +public class MockModelContext implements ModelContext { + private final ApplicationPackage applicationPackage; + + public MockModelContext() { + this.applicationPackage = MockApplicationPackage.createEmpty(); + } + + public MockModelContext(ApplicationPackage applicationPackage) { + this.applicationPackage = applicationPackage; + } + + @Override + public ApplicationPackage applicationPackage() { + return applicationPackage; + } + + @Override + public Optional<Model> previousModel() { + return Optional.empty(); + } + + @Override + public Optional<ApplicationPackage> permanentApplicationPackage() { + return Optional.empty(); + } + + @Override + public Optional<HostProvisioner> hostProvisioner() { + return Optional.empty(); + } + + @Override + public DeployLogger deployLogger() { + return new BaseDeployLogger(); + } + + @Override + public ConfigDefinitionRepo configDefinitionRepo() { + return new StaticConfigDefinitionRepo(); + } + + @Override + public FileRegistry getFileRegistry() { + return new MockFileRegistry(); + } + + @Override + public Properties properties() { + return new Properties() { + @Override + public boolean multitenant() { + return false; + } + + @Override + public ApplicationId applicationId() { + return ApplicationId.defaultId(); + } + + @Override + public List<ConfigServerSpec> configServerSpecs() { + return Collections.emptyList(); + } + + @Override + public boolean hostedVespa() {return false; } + + @Override + public Zone zone() { + return Zone.defaultZone(); + } + + @Override + public Set<Rotation> rotations() { + return new HashSet<>(); + } + }; + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/QrserverAndGatewayPortAllocationTest.java b/config-model/src/test/java/com/yahoo/config/model/QrserverAndGatewayPortAllocationTest.java new file mode 100644 index 00000000000..28a2a163b66 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/QrserverAndGatewayPortAllocationTest.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Tests that qrserver is assigned port Defaults.getDefaults().vespaWebServicePort() even if there is a HTTP gateway configured earlier in + * vespa-services.xml + * + * @author musum + */ +public class QrserverAndGatewayPortAllocationTest { + + @Test + public void testPorts() throws IOException, SAXException { + String appDir = "src/test/cfg/application/app_qrserverandgw/"; + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg(appDir); + VespaModel vespaModel = creator.create(); + List<Container> qrservers = vespaModel.getContainerClusters().get("container").getContainers(); + assertThat(qrservers.size(), is(1)); + assertThat(qrservers.get(0).getSearchPort(), is(Container.BASEPORT)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java new file mode 100644 index 00000000000..63b51acfad5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import org.junit.Test; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.9 + */ +public class SchemaValidatorTest { + + private static final String okServices = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <config name=\"standard\">" + + " <basicStruct>" + + " <stringVal>default</stringVal>" + + " </basicStruct>" + + " </config> " + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + "</services>"; + + private static final String badServices = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <config name=\"standard\">" + + " <basicStruct>" + + " <stringVal>default</stringVal>" + + " </basicStruct>" + + " </config> " + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\"" + + " </admin>" + + "</services>"; + + + @Test + public void testXMLParse() throws SAXException, IOException { + SchemaValidator validator = createValidator(); + validator.validate(new InputSource(new StringReader(okServices)), "services.xml"); + } + + @Test(expected = RuntimeException.class) + public void testXMLParseError() throws SAXException, IOException { + SchemaValidator validator = createValidator(); + validator.validate(new InputSource(new StringReader(badServices)), "services.xml"); + } + + @Test + public void testXMLParseWithReader() throws SAXException, IOException { + SchemaValidator validator = createValidator(); + validator.validate(new StringReader(okServices)); + } + + @Test(expected = RuntimeException.class) + public void testXMLParseErrorWithReader() throws SAXException, IOException { + SchemaValidator validator = createValidator(); + validator.validate(new StringReader(badServices)); + } + + private SchemaValidator createValidator() throws IOException { + return SchemaValidator.createTestValidatorServices(); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/builder/xml/ConfigModelIdTest.java b/config-model/src/test/java/com/yahoo/config/model/builder/xml/ConfigModelIdTest.java new file mode 100644 index 00000000000..e3baa3c1796 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/builder/xml/ConfigModelIdTest.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.Version; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelIdTest { + + @Test + public void require_that_element_gets_correct_name() { + ConfigModelId id = ConfigModelId.fromName("foo"); + assertThat(id.getName(), is("foo")); + assertThat(id.getVersion(), is(Version.fromString("1"))); + id = ConfigModelId.fromNameAndVersion("bar", "2.2"); + assertThat(id.getName(), is("bar")); + assertThat(id.getVersion(), is(Version.fromString("2.2"))); + } + + @Test + public void test_toString() { + ConfigModelId id = ConfigModelId.fromNameAndVersion("bar", "1.0"); + assertThat(id.toString(), is("bar.1")); + id = ConfigModelId.fromNameAndVersion("foo", "1.1.3"); + assertThat(id.toString(), is("foo.1.1.3")); + id = ConfigModelId.fromNameAndVersion("bar", "1"); + assertThat(id.toString(), is("bar.1")); + } + + @Test + public void test_equality() { + ConfigModelId a1 = ConfigModelId.fromName("a"); + ConfigModelId a2 = ConfigModelId.fromName("a"); + ConfigModelId b = ConfigModelId.fromName("b"); + assertTrue(a1.equals(a2)); + assertTrue(a2.equals(a1)); + assertFalse(a1.equals(b)); + assertFalse(a2.equals(b)); + assertFalse(b.equals(a1)); + assertFalse(b.equals(a2)); + assertTrue(a1.equals(a1)); + assertTrue(a2.equals(a2)); + assertTrue(b.equals(b)); + } + + @Test + public void test_compare() { + ConfigModelId a1 = ConfigModelId.fromName("a"); + ConfigModelId a2 = ConfigModelId.fromName("a"); + ConfigModelId b = ConfigModelId.fromName("b"); + assertTrue(a1.compareTo(a2) == 0); + assertTrue(a2.compareTo(a1) == 0); + assertFalse(a1.compareTo(b) > 0); + assertFalse(a2.compareTo(b) > 0); + assertFalse(b.compareTo(a1) < 0); + assertFalse(b.compareTo(a2) < 0); + assertTrue(a1.compareTo(a1) == 0); + assertTrue(a2.compareTo(a2) == 0); + assertTrue(b.compareTo(b) == 0); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/builder/xml/test/DomBuilderTest.java b/config-model/src/test/java/com/yahoo/config/model/builder/xml/test/DomBuilderTest.java new file mode 100644 index 00000000000..ba5e3e4d816 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/builder/xml/test/DomBuilderTest.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml.test; + +import com.yahoo.config.model.test.ConfigModelTestUtil; +import com.yahoo.config.model.test.MockRoot; +import org.junit.Before; +import org.w3c.dom.Element; + +/** + * Utility functions for testing dom builders. + * + * For an example, + * @see com.yahoo.vespa.model.builder.xml.dom.chains.DependenciesBuilderTest + * + * @author tonytv + */ +abstract public class DomBuilderTest { + + public static Element parse(String... xmlLines) { + return ConfigModelTestUtil.parse(xmlLines); + } + + protected MockRoot root; + + @Before + public void setup() { + root = new MockRoot(); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java new file mode 100644 index 00000000000..a84cb7ca7f6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @since 5.12 + */ +public class DeployStateTest { + @Test + public void testProvisionerIsSet() { + DeployState.Builder builder = new DeployState.Builder(); + HostProvisioner provisioner = new InMemoryProvisioner(true, "foo.yahoo.com"); + builder.modelHostProvisioner(provisioner); + DeployState state = builder.build(); + assertThat(state.getProvisioner(), is(provisioner)); + } + + @Test + public void testBuilder() { + DeployState.Builder builder = new DeployState.Builder(); + ApplicationPackage app = MockApplicationPackage.createEmpty(); + builder.permanentApplicationPackage(Optional.of(app)); + DeployState state = builder.build(); + assertThat(state.getPermanentApplicationPackage().get(), is(app)); + } + + @Test + public void testPreviousModelIsProvided() throws IOException, SAXException { + VespaModel prevModel = new VespaModel(MockApplicationPackage.createEmpty()); + DeployState.Builder builder = new DeployState.Builder(); + assertThat(builder.previousModel(prevModel).build().getPreviousModel().get(), is(prevModel)); + } + + @Test + public void testProperties() { + DeployState.Builder builder = new DeployState.Builder(); + DeployState state = builder.build(); + assertThat(state.getProperties().applicationId(), is(ApplicationId.defaultId())); + ApplicationId customId = new ApplicationId.Builder() + .tenant("bar") + .applicationName("foo").instanceName("quux").build(); + DeployProperties properties = new DeployProperties.Builder().applicationId(customId).build(); + builder.properties(properties); + state = builder.build(); + assertThat(state.getProperties().applicationId(), is(customId)); + } + + @Test + public void testDefinitionRepoIsUsed() { + final Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defs = new LinkedHashMap<>(); + defs.put(new ConfigDefinitionKey("foo", "bar"), new com.yahoo.vespa.config.buildergen.ConfigDefinition("foo", new String[]{"namespace=bar", "foo int default=0"})); + defs.put(new ConfigDefinitionKey("test2", "a.b"), + new com.yahoo.vespa.config.buildergen.ConfigDefinition("test2", new String[]{"namespace=a.b", "doubleVal double default=1.0"})); + ApplicationPackage app = FilesApplicationPackage.fromFile(new File("src/test/cfg//application/app1")); + DeployState state = createDeployState(app, defs); + + assertNotNull(state.getConfigDefinition(new ConfigDefinitionKey("foo", "bar"))); + assertNotNull(state.getConfigDefinition(new ConfigDefinitionKey("test1", ""))); + ConfigDefinition overridden = state.getConfigDefinition(new ConfigDefinitionKey("test2", "a.b")); + assertNotNull(overridden); + Double defaultValue = overridden.getDoubleDefs().get("doubleVal").getDefVal(); + assertNotNull(defaultValue); + assertThat(defaultValue.intValue(), is(0)); + } + + @Test + public void testGetConfigDefinition() { + final Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defs = new LinkedHashMap<>(); + defs.put(new ConfigDefinitionKey("test2", "a.b"), new com.yahoo.vespa.config.buildergen.ConfigDefinition("test2", new String[]{"namespace=a.b", "doubleVal double default=1.0"})); + defs.put(new ConfigDefinitionKey("test2", "c.d"), new com.yahoo.vespa.config.buildergen.ConfigDefinition("test2", new String[]{"namespace=c.d", "doubleVal double default=1.0"})); + defs.put(new ConfigDefinitionKey("test3", "xyzzy"), new com.yahoo.vespa.config.buildergen.ConfigDefinition("test3", new String[]{"namespace=xyzzy", "message string"})); + ApplicationPackage app = FilesApplicationPackage.fromFile(new File("src/test/cfg//application/app1")); + DeployState state = createDeployState(app, defs); + + assertNotNull(state.getConfigDefinition(new ConfigDefinitionKey("test2", "a.b"))); + + // Should not fallback to using test2 with another namespace + try { + state.getConfigDefinition(new ConfigDefinitionKey("test2", "")); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Using config definition 'test2' is ambiguous, there are more than one config definitions with this name, please specify namespace")); + } + + final ConfigDefinition test1 = state.getConfigDefinition(new ConfigDefinitionKey("test2", "a.b")); + assertNotNull(test1); + assertThat(test1.getName(), is("test2")); + assertThat(test1.getNamespace(), is("a.b")); + + // Should fallback to using test3 with another namespace, since only one exists + ConfigDefinition test3 = state.getConfigDefinition(new ConfigDefinitionKey("test3", "")); + assertNotNull(test3); + assertThat(test3.getName(), is("test3")); + assertThat(test3.getNamespace(), is("xyzzy")); + } + + @Test + public void testRotations() { + final Set<Rotation> rotations = new HashSet<>(); + assertThat(new DeployState.Builder().rotations(rotations).build().getRotations().size(), is(0)); + for (String name : new String[]{"rotation-001.vespa.a02.yahoodns.net", "rotation-002.vespa.a02.yahoodns.net"}) { + rotations.add(new Rotation(name)); + } + assertThat(new DeployState.Builder().rotations(rotations).build().getRotations(), equalTo(rotations)); + } + + private DeployState createDeployState(ApplicationPackage app, final Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defs) { + DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); + builder.configDefinitionRepo(new ConfigDefinitionRepo() { + @Override + public Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> getConfigDefinitions() { + return defs; + } + }); + return builder.build(); + } +} + diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java new file mode 100644 index 00000000000..6be85fb19e4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.test.StandardConfig; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.net.HostName; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.test.ApiConfigModel; +import com.yahoo.vespa.model.test.SimpleConfigModel; +import com.yahoo.vespa.model.test.SimpleService; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class SystemModelTestCase { + + private static final String TESTDIR = "src/test/cfg/application/"; + + private static VespaModel getVespaModelDoNotValidateXml(String configPath) { + ConfigModelRegistry registry = MapConfigModelRegistry.createFromList(new SimpleConfigModel.Builder(), new ApiConfigModel.Builder()); + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg(configPath,registry); + return creator.create(false); // do not validate against schema -- the xml files used here are not valid + } + + // Debugging + @SuppressWarnings({"UnusedDeclaration"}) + private void dumpTree(ConfigProducer producer) { + Map<String,? extends ConfigProducer> id2cp = producer.getChildren(); + for (ConfigProducer c : id2cp.values()) { + System.out.println("id: " + c.getConfigId()); + if (c.getChildren().size() > 0) { + dumpTree(c); + } + } + } + + @Test + public void testMetrics() { + VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "metricsconfig"); + SimpleService service0 = (SimpleService)vespaModel.getConfigProducer("simple/simpleservice.0").get(); + vespaModel.getConfigProducer("simple/simpleservice.1"); + assertThat(service0.getDefaultMetricDimensions().get("clustername"), is("testClusterName")); + } + + @Test + public void testVespaModel() { + VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); + assertNotNull(vespaModel); + + assertEquals("There are two instances of the simple model + Routing and AdminModel (set up implicitly)", 4, vespaModel.configModelRepo().asMap().size()); + assertNotNull("One gets the default name as there is no explicit id", vespaModel.configModelRepo().asMap().get("simple")); + assertNotNull("The other gets the explicit id as name", vespaModel.configModelRepo().asMap().get("second")); + + ApplicationConfigProducerRoot root = vespaModel.getVespa(); + assertNotNull(root); + + // Verify configIds from vespa + assertTrue(6 <= root.getConfigIds().size()); + assertTrue(root.getConfigIds().contains("client")); + assertTrue(root.getConfigIds().contains("simple")); + assertTrue(root.getConfigIds().contains("second")); + assertTrue(root.getConfigIds().contains("simple/simpleservice.0")); + assertTrue(root.getConfigIds().contains("simple/simpleservice.1")); + assertTrue(root.getConfigIds().contains("second/simpleservice.0")); + + // Verify configIds from vespaModel + assertTrue(12 <= vespaModel.getConfigIds().size()); + String localhost = HostName.getLocalhost(); + String localhostConfigId = "hosts/" + localhost; + Set<String> configIds = vespaModel.getConfigIds(); + assertTrue(configIds.contains("client")); + assertTrue(configIds.contains(localhostConfigId)); + assertTrue(configIds.contains("simple/simpleservice.0")); + assertTrue(configIds.contains("second/simpleservice.0")); + assertTrue(configIds.contains("hosts/" + localhost + "/logd")); + + // Verify sentinel config + SentinelConfig sentinelConfig = new SentinelConfig((SentinelConfig.Builder) vespaModel.getConfig(new SentinelConfig.Builder(), localhostConfigId)); + boolean found = false; + for (SentinelConfig.Service service : sentinelConfig.service()) { + if ("logd".equals(service.name())) { + found = true; + } + } + assertTrue(found); + + // Get the simple model config from VespaModel + assertEquals(vespaModel.getConfig(StandardConfig.class, "simple/simpleservice.0").astring(), "simpleservice"); + assertEquals(vespaModel.getConfig(StandardConfig.class, "second/simpleservice.0").astring(), "simpleservice"); + } + + @Test + public void testHostSystem() { + VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); + HostSystem hostSystem = vespaModel.getHostSystem(); + + HostResource host1 = hostSystem.getHost("host1"); + HostResource host2 = hostSystem.getHost("host2"); + HostResource host3 = hostSystem.getHost("host3"); + + assertEquals(host1, host2); + assertEquals(host2, host3); + + // all three host aliases are for the same host, so the number of services should be 3 + 8 + // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, log server and file distribution) + assertEquals(10, host1.getServices().size()); + + assertNotNull(host1.getService("simpleservice")); + assertNotNull(host1.getService("simpleservice2")); + assertNotNull(host3.getService("simpleservice3")); + } + + @Test + public void testBasePorts() { + VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig"); + assertNotNull(vespaModel); + + assertEquals(vespaModel.getConfig(StandardConfig.class, "simple/simpleservice.0").baseport(), 10000); + assertTrue(vespaModel.getConfig(StandardConfig.class, "simple/simpleservice.1").baseport() != 10000); + } + + /** + * This test is the same as the system test cloudconfig/plugins. + * Be sure to update it as well if you change this. + */ + @Test + public void testPlugins() { + VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "plugins"); + + assertNotNull(vespaModel); + ApplicationConfigProducerRoot root = vespaModel.getVespa(); + + assertEquals(5, vespaModel.configModelRepo().asMap().size()); + assertTrue(vespaModel.configModelRepo().asMap().keySet().contains("simple")); + assertTrue(vespaModel.configModelRepo().asMap().keySet().contains("api")); + assertTrue(root.getConfigIds().contains("simple/simpleservice.0")); + assertTrue(root.getConfigIds().contains("simple/simpleservice.1")); + assertTrue(root.getConfigIds().contains("api/apiservice.0")); + + // Verify that configModelRegistry iterates in dependency order + Iterator<ConfigModel> i = vespaModel.configModelRepo().iterator(); + ConfigModel plugin = i.next(); + assertEquals("admin", plugin.getId()); + plugin = i.next(); + assertEquals("simple", plugin.getId()); + plugin = i.next(); + assertEquals("simple2", plugin.getId()); + plugin = i.next(); + assertEquals("api", plugin.getId()); + plugin = i.next(); + assertEquals("routing", plugin.getId()); + + assertEquals(vespaModel.getConfig(StandardConfig.class, "api/apiservice.0").astring(), "apiservice"); + + assertEquals(vespaModel.getConfig(StandardConfig.class, "simple/simpleservice.0").astring(), "simpleservice"); + assertEquals(vespaModel.getConfig(StandardConfig.class, "simple/simpleservice.1").astring(), "simpleservice"); + assertEquals(vespaModel.getConfig(StandardConfig.class, "simple2/simpleservice.0").astring(), "simpleservice"); + } + + @Test + public void testEqualPlugins() { + try { + getVespaModelDoNotValidateXml(TESTDIR + "doubleconfig"); + fail("No exception upon two plugins with the same name"); + } catch (RuntimeException expected) { + assertThat(expected.getMessage(), is("Could not resolve tag <simpleplugin version=\"1.0\"> to a config model component")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java b/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java new file mode 100644 index 00000000000..2fd8115a804 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.google.inject.Inject; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * @author lulf + * @since 5.1 + */ +public class GraphMock { + + public static class BA extends ConfigModelBuilder<A> { + public BA() { super(A.class); } + @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public void doBuild(A model, Element spec, ConfigModelContext modelContext) { } + } + public static class A extends ConfigModel { + public A(ConfigModelContext modelContext) { super(modelContext); } + } + + public static class BB extends ConfigModelBuilder<B> { + public BB() { super(B.class); } + @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public void doBuild(B model, Element spec, ConfigModelContext modelContext) { } + } + public static class B extends ConfigModel { + public final A a; + @Inject + public B(ConfigModelContext modelContext, A modelA) { super(modelContext); this.a = modelA; } + } + + public static class BC extends ConfigModelBuilder<C> { + public BC() { super(C.class); } + @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public void doBuild(C model, Element spec, ConfigModelContext modelContext) { } + } + public static class C extends ConfigModel { + public Collection<B> b; + public A a; + public C(ConfigModelContext modelContext, Collection<B> modelB, A modelA) { super(modelContext); b = modelB; a = modelA; } + } + + public static class BD extends ConfigModelBuilder<D> { + public BD() { super(D.class); } + @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public void doBuild(D model, Element spec, ConfigModelContext modelContext) { } + } + public static class D extends ConfigModel { + public D(ConfigModelContext modelContext, E modelE) { super(modelContext); } + } + + public static class BE extends ConfigModelBuilder<E> { + public BE() { super(E.class); } + @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public void doBuild(E model, Element spec, ConfigModelContext modelContext) { } + } + public static class E extends ConfigModel { + public E(ConfigModelContext modelContext, D modelD) { super(modelContext); } + } + + public static class Bad extends ConfigModel { + public Bad() { super(null); } + public static class Builder extends ConfigModelBuilder<Bad> { + public Builder() { super(Bad.class); } + @Override public List<ConfigModelId> handlesElements() { return null; } + @Override public void doBuild(Bad model, Element spec, ConfigModelContext modelContext) { } + } + } + + public static class Bad2 extends ConfigModel { + public Bad2(ConfigModelContext ctx, String foo) { super(ctx); } + public static class Builder extends ConfigModelBuilder<Bad2> { + public Builder() { super(Bad2.class); } + @Override public List<ConfigModelId> handlesElements() { return null; } + @Override public void doBuild(Bad2 model, Element spec, ConfigModelContext modelContext) { } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/graph/ModelGraphTest.java b/config-model/src/test/java/com/yahoo/config/model/graph/ModelGraphTest.java new file mode 100644 index 00000000000..6970c27eca2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/graph/ModelGraphTest.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.test.MockRoot; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class ModelGraphTest { + + void assertOrdering(ModelGraph graph, String expectedOrdering) { + List<ModelNode> sortedEntries = graph.topologicalSort(); + StringBuilder sb = new StringBuilder(); + for (ModelNode<?> node : sortedEntries) { + sb.append(node.builder.getModelClass().getSimpleName()); + } + assertThat(sb.toString(), is(expectedOrdering)); + } + + @Test + public void require_that_dependencies_are_correctly_set() { + ModelGraphBuilder builder = new ModelGraphBuilder(); + builder.addBuilder(new GraphMock.BC()).addBuilder(new GraphMock.BB()).addBuilder(new GraphMock.BA()); + ModelGraph graph = builder.build(); + List<ModelNode> nodes = graph.getNodes(); + assertThat(graph.getNodes().size(), is(3)); + assertTrue(nodes.get(0).hasDependencies()); + assertTrue(nodes.get(1).hasDependencies()); + assertFalse(nodes.get(2).hasDependencies()); + assertTrue(nodes.get(0).dependsOn(nodes.get(1))); + assertTrue(nodes.get(1).dependsOn(nodes.get(2))); + assertFalse(nodes.get(2).dependsOn(nodes.get(0))); + } + + @Test + public void require_that_dependencies_are_correctly_sorted() { + ModelGraph graph = new ModelGraphBuilder().addBuilder(new GraphMock.BC()).addBuilder(new GraphMock.BB()).addBuilder(new GraphMock.BA()).build(); + assertOrdering(graph, "ABC"); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_cycles_are_detected() { + ModelGraph graph = new ModelGraphBuilder().addBuilder(new GraphMock.BD()).addBuilder(new GraphMock.BE()).build(); + assertThat(graph.getNodes().size(), is(2)); + assertTrue(graph.getNodes().get(0).dependsOn(graph.getNodes().get(1))); + assertTrue(graph.getNodes().get(1).dependsOn(graph.getNodes().get(0))); + graph.topologicalSort(); + } + + @Test + public void require_that_instance_can_be_created() { + ModelGraph graph = new ModelGraphBuilder().addBuilder(new GraphMock.BC()).addBuilder(new GraphMock.BB()).addBuilder(new GraphMock.BA()).build(); + List<ModelNode> nodes = graph.topologicalSort(); + MockRoot root = new MockRoot(); + GraphMock.A a = (GraphMock.A) nodes.get(0).createModel(ConfigModelContext.createFromParentAndId(null, root, "first")); + GraphMock.B b = (GraphMock.B) nodes.get(1).createModel(ConfigModelContext.createFromParentAndId(null, root, "second")); + GraphMock.B b2 = (GraphMock.B) nodes.get(1).createModel(ConfigModelContext.createFromParentAndId(null, root, "second2")); + GraphMock.C c = (GraphMock.C) nodes.get(2).createModel(ConfigModelContext.createFromParentAndId(null, root, "third")); + assertNotNull(a); + assertNotNull(b); + assertNotNull(b2); + assertNotNull(c); + assertThat(a.getId(), is("first")); + assertThat(b.getId(), is("second")); + assertThat(b2.getId(), is("second2")); + assertThat(c.getId(), is("third")); + assertThat(b.a, is(a)); + assertNotNull(c.b); + assertThat(c.b.size(), is(2)); + assertTrue(c.b.contains(b)); + assertTrue(c.b.contains(b2)); + for (ConfigModel m : c.b) { + System.out.println(m.getId()); + } + } + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void require_that_context_must_be_first_ctor_param() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Constructor for " + GraphMock.Bad.class.getName() + " must have as its first argument a " + ConfigModelContext.class.getName()); + ModelNode node = new ModelNode(new GraphMock.Bad.Builder()); + node.createModel(ConfigModelContext.createFromParentAndId(null, new MockRoot(), "foo")); + } + + @Test + public void require_that_ctor_arguments_must_be_models_or_collections_of_models() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Unable to find constructor argument class java.lang.String for com.yahoo.config.model.graph.GraphMock$Bad2"); + ModelNode node = new ModelNode(new GraphMock.Bad2.Builder()); + node.createModel(ConfigModelContext.createFromParentAndId(null, new MockRoot(), "foo")); + } + + @Test + public void require_that_collections_can_be_empty() { + ModelGraph graph = new ModelGraphBuilder().addBuilder(new GraphMock.BC()).addBuilder(new GraphMock.BA()).build(); + List<ModelNode> nodes = graph.topologicalSort(); + MockRoot root = new MockRoot(); + GraphMock.A a = (GraphMock.A) nodes.get(0).createModel(ConfigModelContext.createFromParentAndId(null, root, "first")); + GraphMock.C c = (GraphMock.C) nodes.get(1).createModel(ConfigModelContext.createFromParentAndId(null, root, "second")); + assertThat(c.a, is(a)); + assertTrue(c.b.isEmpty()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java b/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java new file mode 100644 index 00000000000..86b0a9d05d6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.yahoo.cloud.config.log.LogdConfig; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * Verifies some of the logic in the abstract config producer that is not tested in other classes. + * + * @author lulf + * @since 5.1 + */ +public class AbstractConfigProducerTest { + + @Test + public void require_that_interface_is_found_if_directly_implemented() throws ClassNotFoundException, IllegalAccessException, InstantiationException { + MockLogdProducer producer = new MockLogdProducer("mocky"); + ClassLoader loader = producer.getConfigClassLoader(LogdConfig.Producer.class.getName()); + assertNotNull(loader); + Class clazz = loader.loadClass(LogdConfig.Builder.class.getName()); + LogdConfig.Builder builder = (LogdConfig.Builder) clazz.newInstance(); + producer.getConfig(builder); + LogdConfig config = new LogdConfig(builder); + assertThat(config.logserver().host(), is("bar")); + assertThat(config.logserver().port(), is(1338)); + } + + @Test + public void require_that_interface_is_found_if_inherited() throws ClassNotFoundException, IllegalAccessException, InstantiationException { + MockLogdProducerSubclass producer = new MockLogdProducerSubclass("mocky"); + ClassLoader loader = producer.getConfigClassLoader(LogdConfig.Producer.class.getName()); + assertNotNull(loader); + Class clazz = loader.loadClass(LogdConfig.Builder.class.getName()); + LogdConfig.Builder builder = (LogdConfig.Builder) clazz.newInstance(); + producer.getConfig(builder); + LogdConfig config = new LogdConfig(builder); + assertThat(config.logserver().host(), is("foo")); + assertThat(config.logserver().port(), is(1337)); + } + + private static class MockLogdProducer extends AbstractConfigProducer implements LogdConfig.Producer { + + public MockLogdProducer(String subId) { + super(subId); + } + + @Override + public void getConfig(LogdConfig.Builder builder) { + builder.logserver(new LogdConfig.Logserver.Builder().host("bar").port(1338)); + } + } + + private static abstract class MockLogdSuperClass extends AbstractConfigProducer implements LogdConfig.Producer { + + public MockLogdSuperClass(String subId) { + super(subId); + } + } + + private static class MockLogdProducerSubclass extends MockLogdSuperClass { + public MockLogdProducerSubclass(String subId) { + super(subId); + } + + @Override + public void getConfig(LogdConfig.Builder builder) { + builder.logserver(new LogdConfig.Logserver.Builder().host("foo").port(1337)); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java new file mode 100644 index 00000000000..7a8e30d4a66 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.provision.HostSpec; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.11 + */ +public class HostSpecTest { + @Test + public void testEquals() { + HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList()); + HostSpec h2 = new HostSpec("foo", Collections.<String>emptyList()); + HostSpec h3 = new HostSpec("foo", Arrays.asList("my", "alias")); + HostSpec h4 = new HostSpec("bar", Collections.<String>emptyList()); + + assertTrue(h1.equals(h1)); + assertTrue(h1.equals(h2)); + assertTrue(h1.equals(h3)); + assertFalse(h1.equals(h4)); + + assertTrue(h2.equals(h1)); + assertTrue(h2.equals(h2)); + assertTrue(h2.equals(h3)); + assertFalse(h2.equals(h4)); + + assertTrue(h3.equals(h1)); + assertTrue(h3.equals(h2)); + assertTrue(h3.equals(h3)); + assertFalse(h3.equals(h4)); + + assertFalse(h4.equals(h1)); + assertFalse(h4.equals(h2)); + assertFalse(h4.equals(h3)); + assertTrue(h4.equals(h4)); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java new file mode 100644 index 00000000000..2a0a0bf224b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.provision.HostSpec; +import com.yahoo.vespa.model.container.Container; +import org.junit.Test; + +import java.io.StringReader; +import java.util.*; + +import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * @author musum + */ +public class HostsXmlProvisionerTest { + private static final String oneHost = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + + "<hosts>\n" + + " <host name=\"test1.yahoo.com\">\n" + + " <alias>node1</alias>\n" + + " <alias>node2</alias>\n" + + " </host>\n" + + "</hosts>"; + + private static final String threeHosts = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + + "<hosts>\n" + + " <host name=\"test1.yahoo.com\">\n" + + " <alias>node1</alias>\n" + + " </host>\n" + + " <host name=\"test2.yahoo.com\">\n" + + " <alias>node2</alias>\n" + + " <alias>node3</alias>\n" + + " </host>\n" + + " <host name=\"test3.yahoo.com\">\n" + + " <alias>node4</alias>\n" + + " </host>\n" + + "</hosts>"; + + @Test + public void require_basic_works() { + HostsXmlProvisioner hostProvisioner = createProvisioner(threeHosts); + + // 4 services, 2 host aliases, mapping to 2 host. + List<String> aliases = createAliases(); + Map<String, HostSpec> map = allocate(hostProvisioner, aliases); + + assertCorrectNumberOfHosts(map, 2); + for (HostSpec hostSpec : map.values()) { + if (hostSpec.hostname().equals("test2.yahoo.com")) { + assertThat(hostSpec.aliases().size(), is(2)); + } else { + assertThat(hostSpec.aliases().size(), is(1)); + } + } + assertThat(map.size(), is(2)); + assertTrue(map.keySet().containsAll(aliases)); + + // 5 services, 3 host aliases, mapping to 2 host. + aliases = createAliases(Collections.singletonList("node3")); + map = allocate(hostProvisioner, aliases); + + assertCorrectNumberOfHosts(map, 2); + assertThat(map.size(), is(3)); + assertTrue(map.keySet().containsAll(aliases)); + + // 5 services, 3 host aliases, mapping to 3 host. + aliases = createAliases(Collections.singletonList("node4")); + map = allocate(hostProvisioner, aliases); + assertThat(map.size(), is(3)); + assertCorrectNumberOfHosts(map, 3); + assertTrue(map.keySet().containsAll(aliases)); + } + + @Test(expected = IllegalArgumentException.class) + public void require_exception_when_unknown_hosts_alias() { + HostsXmlProvisioner hostProvisioner = createProvisioner(oneHost); + hostProvisioner.allocateHost("unknown"); + } + + private void assertCorrectNumberOfHosts(Map<String, HostSpec> hostToServiceMap, int expectedHostCount) { + Set<String> hostSet = new HashSet<>(); + for (HostSpec host : hostToServiceMap.values()) { + hostSet.add(host.hostname()); + } + assertThat(hostSet.size(), is(expectedHostCount)); + } + + private HostsXmlProvisioner createProvisioner(String hosts) { + return new HostsXmlProvisioner(new StringReader(hosts)); + } + + private List<String> createAliases() { + return createAliases(new ArrayList<>()); + } + + // Admin services on node1, qrserver on node2 + additional specs + private List<String> createAliases(Collection<String> additionalAliases) { + ArrayList<String> aliases = new ArrayList<>(); + aliases.add("node1"); + aliases.add("node1"); + aliases.add("node1"); + aliases.add("node2"); + aliases.addAll(additionalAliases); + return aliases; + } + + private Map<String, HostSpec> allocate(HostsXmlProvisioner hostProvisioner, List<String> aliases) { + Map<String, HostSpec> map = new LinkedHashMap<>(); + for (String alias : aliases) { + map.put(alias, hostProvisioner.allocateHost(alias)); + } + return map; + } + + @Test + public void require_singlenode_HostAlias_is_used_if_hosts_xml() { + String servicesXml = "<jdisc id='default' version='1.0' />"; + HostsXmlProvisioner hostProvisioner = createProvisioner(oneHost); + HostSpec hostSpec = hostProvisioner.allocateHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + assertThat(hostSpec.hostname(), is("test1.yahoo.com")); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java new file mode 100644 index 00000000000..b74398fc4ae --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -0,0 +1,1108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +import java.io.StringReader; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.Slobrok; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.search.Dispatch; +import org.junit.Test; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +/** + * Test cases for provisioning nodes to entire vespamodels + * + * @author vegardh + * @author bratseth + */ +public class ModelProvisioningTest { + + private Hosts createHosts(int n) { return createHosts("", n); } + private Hosts createHosts(String hostnamePrefix, int n) { + Hosts hosts = new Hosts(); + if ( ! hostnamePrefix.isEmpty()) + hostnamePrefix = "-" + hostnamePrefix; + for (int i = 0; i < n; i++) + hosts.addHost(new com.yahoo.config.model.provision.Host(hostnamePrefix + "foo" + i), Collections.emptyList()); + return hosts; + } + + /** Creates a model with hosts of the 'default' flavor */ + private VespaModel createModel(String services, Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) throws ParseException { + return createModel(services, Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames); + } + + /** + * Creates a model + * + * @param services the services xml string + * @param hosts hosts by flavor + * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor + * is available or if we should just silently receive a smaller allocation + * @return the resulting model + * @throws ParseException if the services xml is invalid + */ + private VespaModel createModel(String services, Map<String, Collection<Host>> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) throws ParseException { + final VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + final ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; + DeployState deployState = new DeployState.Builder().applicationPackage(appPkg).modelHostProvisioner(new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames)). + properties((new DeployProperties.Builder()).hostedVespa(true).build()).build(); + return modelCreatorWithMockPkg.create(false, deployState); + } + + private void assertCorrectModel(VespaModel model, int numberOfHosts, int numberOfContentNodes) { + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + final Map<String, ContentCluster> contentClusters = model.getContentClusters(); + ContentCluster cluster = contentClusters.get("bar"); + assertThat(cluster.getRootGroup().getNodes().size(), is(numberOfContentNodes)); + int i = 0; + for (StorageNode node : cluster.getRootGroup().getNodes()) { + assertThat(node.getDistributionKey(), is(i)); + i++; + } + } + + @Test + public void testNodeCountForJdisc() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>\n" + + "\n" + + "<admin version='3.0'><nodes count='1' /></admin>\n" + + "<jdisc id='mydisc' version='1.0'>" + + " <handler id='myHandler'>" + + " <component id='injected' />" + + " </handler>" + + " <nodes count=\"3\"/>" + + "</jdisc>" + + "<jdisc id='mydisc2' version='1.0'>" + + " <handler id='myHandler'>" + + " <component id='injected' />" + + " </handler>" + + " <nodes count='2' jvmargs='-verbosegc' preload='lib/blablamalloc.so'/>" + + "</jdisc>" + + "</services>"; + String hosts ="<hosts>" + + " <host name='myhost0'>" + + " <alias>node0</alias>" + + " </host>" + + " <host name='myhost1'>" + + " <alias>node1</alias>" + + " </host>" + + " <host name='myhost2'>" + + " <alias>node2</alias>" + + " </host>" + + " <host name='myhost3'>" + + " <alias>node3</alias>" + + " </host>" + + " <host name='myhost4'>" + + " <alias>node4</alias>" + + " </host>" + + " <host name='myhost5'>" + + " <alias>node5</alias>" + + " </host>" + + "</hosts>"; + VespaModelCreatorWithMockPkg creator = new VespaModelCreatorWithMockPkg(null, services); + VespaModel model = creator.create(new DeployState.Builder().modelHostProvisioner(new InMemoryProvisioner(Hosts.getHosts(new StringReader(hosts)), true))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().size(), is(3)); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getConfigId(), is("mydisc/container.0")); + assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(0).isInitialized()); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getConfigId(), is("mydisc/container.1")); + assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(1).isInitialized()); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getConfigId(), is("mydisc/container.2")); + assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(2).isInitialized()); + + assertThat(model.getContainerClusters().get("mydisc2").getContainers().size(), is(2)); + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getConfigId(), is("mydisc2/container.0")); + assertTrue(model.getContainerClusters().get("mydisc2").getContainers().get(0).isInitialized()); + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getConfigId(), is("mydisc2/container.1")); + assertTrue(model.getContainerClusters().get("mydisc2").getContainers().get(1).isInitialized()); + + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getJvmArgs(), is("")); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getJvmArgs(), is("")); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getJvmArgs(), is("")); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so")); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so")); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so")); + + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc")); + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getJvmArgs(), is("-verbosegc")); + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getPreLoad(), is("lib/blablamalloc.so")); + assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getPreLoad(), is("lib/blablamalloc.so")); + + final HostSystem hostSystem = model.getHostSystem(); + assertNotNull(hostSystem.getHostByHostname("myhost0")); + assertNotNull(hostSystem.getHostByHostname("myhost1")); + assertNotNull(hostSystem.getHostByHostname("myhost2")); + assertNotNull(hostSystem.getHostByHostname("myhost3")); + assertNull(hostSystem.getHostByHostname("Nope")); + } + + @Test + public void testNodeCountForContentGroup() throws Exception { + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + "\n" + + " <admin version='3.0'>" + + " <nodes count='3'/>" + + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + int numberOfHosts = 2; + Hosts hosts = createHosts(numberOfHosts); + int numberOfContentNodes = 2; + VespaModel model = createModel(xmlWithNodes, hosts, true); + assertCorrectModel(model, numberOfHosts, numberOfContentNodes); + } + + @Test + public void testNodeCountForContentGroupHierarchy() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>\n" + + "\n" + + " <admin version='3.0'>\n" + + " <nodes count='3'/>" + // Ignored + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>\n" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <group>" + + " <distribution partitions=\"1|*\"/>" + + " <group name='0' distribution-key='0'>" + + " <nodes count='2'/> " + + " </group>" + + " <group name='1' distribution-key='1'>" + + " <nodes count='2'/> " + + " </group>" + + " </group>" + + " </content>" + + " <content version='1.0' id='baz'>" + + " <redundancy>2</redundancy>\n" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <group>" + + " <distribution partitions=\"1|*\"/>" + + " <group name='0' distribution-key='10'>" + + " <nodes count='1'/> " + + " </group>" + + " <group name='1' distribution-key='11'>" + + " <nodes count='1'/> " + + " </group>" + + " </group>" + + " </content>" + + "\n" + + "</services>"; + + int numberOfHosts = 6; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertThat(cluster.getRootGroup().getNodes().size(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/2")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/3")); + + cluster = model.getContentClusters().get("baz"); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("10")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("11")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1")); + } + + @Test + public void testUsingNodesAndGroupCountAttributes() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='27' groups='9'/>" + + " </content>" + + " <content version='1.0' id='baz'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='27' groups='27'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 64; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check container cluster + assertEquals(1, model.getContainerClusters().size()); + Set<com.yahoo.vespa.model.Host> containerHosts = model.getContainerClusters().get("foo").getContainers().stream().map(Container::getHost).collect(Collectors.toSet()); + assertEquals(10, containerHosts.size()); + + // Check admin clusters + Admin admin = model.getAdmin(); + Set<com.yahoo.vespa.model.Host> slobrokHosts = admin.getSlobroks().stream().map(Slobrok::getHost).collect(Collectors.toSet()); + assertEquals(3, slobrokHosts.size()); + assertTrue("Slobroks are assigned from container nodes", containerHosts.containsAll(slobrokHosts)); + assertTrue("Logserver is assigned from container nodes", containerHosts.contains(admin.getLogserver().getHost())); + assertEquals("No in-cluster config servers in a hosted environment", 0, admin.getConfigservers().size()); + assertEquals("No admin cluster controller when multitenant", null, admin.getClusterControllers()); + + // Check content clusters + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(3, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("foo10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("foo13", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("foo16", clusterControllers.getContainers().get(2).getHostName()); + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(9, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); + assertEquals("foo10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getDistributionKey(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(2).getConfigId(), is("bar/storage/2")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/3")); + assertEquals("foo13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(4)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/4")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getDistributionKey(), is(5)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(2).getConfigId(), is("bar/storage/5")); + // ... + assertEquals("foo16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + // ... + assertThat(cluster.getRootGroup().getSubgroups().get(8).getIndex(), is("8")); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().size(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(0).getDistributionKey(), is(24)); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(0).getConfigId(), is("bar/storage/24")); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(1).getDistributionKey(), is(25)); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(1).getConfigId(), is("bar/storage/25")); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getDistributionKey(), is(26)); + assertThat(cluster.getRootGroup().getSubgroups().get(8).getNodes().get(2).getConfigId(), is("bar/storage/26")); + + cluster = model.getContentClusters().get("baz"); + clusterControllers = cluster.getClusterControllers(); + assertEquals(3, clusterControllers.getContainers().size()); + assertEquals("baz-controllers", clusterControllers.getName()); + assertEquals("foo37", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("foo38", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("foo39", clusterControllers.getContainers().get(2).getHostName()); + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(27, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("baz/storage/0")); + assertEquals("foo37", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("baz/storage/1")); + assertEquals("foo38", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + // ... + assertEquals("foo39", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + // ... + assertThat(cluster.getRootGroup().getSubgroups().get(26).getIndex(), is("26")); + assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().get(0).getDistributionKey(), is(26)); + assertThat(cluster.getRootGroup().getSubgroups().get(26).getNodes().get(0).getConfigId(), is("baz/storage/26")); + } + + @Test + public void testGroupsOfSize1() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='8' groups='8'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 18; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check content cluster + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(3, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("foo10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("foo11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("foo12", clusterControllers.getContainers().get(2).getHostName()); + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(8, cluster.getRootGroup().getSubgroups().size()); + assertEquals(8, cluster.distributionBits()); + // first group + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); + assertEquals("foo10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + // second group + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/1")); + assertEquals("foo11", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + // ... last group + assertThat(cluster.getRootGroup().getSubgroups().get(7).getIndex(), is("7")); + assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getDistributionKey(), is(7)); + assertThat(cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getConfigId(), is("bar/storage/7")); + assertEquals("foo17", cluster.getRootGroup().getSubgroups().get(7).getNodes().get(0).getHostName()); + } + + @Test + public void testExplicitNonDedicatedClusterControllers() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <controllers><nodes dedicated='false' count='6'/></controllers>" + + " <nodes count='9' groups='3'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 19; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check content clusters + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals( 8, cluster.distributionBits()); + assertEquals("We get the closest odd numer", 5, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("foo10", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("foo11", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("foo13", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("foo14", clusterControllers.getContainers().get(3).getHostName()); // Should be 16 for perfect distribution ... + assertEquals("foo10", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getHostName()); + assertEquals("foo11", cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getHostName()); + assertEquals("foo13", cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getHostName()); + assertEquals("foo16", cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getHostName()); + } + + @Test + public void testClusterControllersAreNotPlacedOnRetiredNodes() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='9' groups='3'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 19; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true, "foo10", "foo13", "foo16"); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check content clusters + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(3, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("Skipping retired foo10", "foo11", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("Skipping retired foo13", "foo14", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("Skipping retired foo16", "foo17", clusterControllers.getContainers().get(2).getHostName()); + } + + @Test + public void testSlobroksClustersAreExpandedToIncludeRetiredNodes() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 10; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true, "foo0"); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check slobroks clusters + assertEquals("Includes retired node", 1+3, model.getAdmin().getSlobroks().size()); + assertEquals("foo1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("foo2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("foo3", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "foo0", model.getAdmin().getSlobroks().get(3).getHostName()); + } + + @Test + public void testSlobroksClustersAreExpandedToIncludeRetiredNodesWhenRetiredComesLast() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 10; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true, "foo3", "foo4"); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check slobroks clusters + assertEquals("Includes retired node", 3+2, model.getAdmin().getSlobroks().size()); + assertEquals("foo0", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("foo1", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("foo2", model.getAdmin().getSlobroks().get(2).getHostName()); + assertEquals("Included in addition because it is retired", "foo3", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "foo4", model.getAdmin().getSlobroks().get(4).getHostName()); + } + + @Test + public void testSlobroksAreSpreadOverAllContainerClusters() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <container version='1.0' id='bar'>" + + " <nodes count='3'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 13; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true, "foo0", "foo10", "foo11"); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check slobroks clusters + // ... from cluster foo + assertEquals("Includes retired node", 3+3, model.getAdmin().getSlobroks().size()); + assertEquals("foo1", model.getAdmin().getSlobroks().get(0).getHostName()); + assertEquals("foo2", model.getAdmin().getSlobroks().get(1).getHostName()); + assertEquals("Included in addition because it is retired", "foo0", model.getAdmin().getSlobroks().get(2).getHostName()); + // ... from cluster bar + assertEquals("foo12", model.getAdmin().getSlobroks().get(3).getHostName()); + assertEquals("Included in addition because it is retired", "foo10", model.getAdmin().getSlobroks().get(4).getHostName()); + assertEquals("Included in addition because it is retired", "foo11", model.getAdmin().getSlobroks().get(5).getHostName()); + } + + @Test + public void test2ContentNodesProduces1ClusterController() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 2; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(1, clusterControllers.getContainers().size()); + } + + @Test + public void testExplicitDedicatedClusterControllers() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <container version='1.0' id='foo'>" + + " <nodes count='10'/>" + + " </container>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <controllers><nodes dedicated='true' count='4'/></controllers>" + + " <nodes count='9' groups='3'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 23; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + // Check content clusters + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(4, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("foo19", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("foo20", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("foo21", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("foo22", clusterControllers.getContainers().get(3).getHostName()); + } + + @Test + public void testUsingNodesAndGroupCountAttributesAndGettingTooFewNodes() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <admin version='3.0'>" + + " <nodes count='3'/>" + // Ignored + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy reply-after='3'>4</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='24' groups='3'/>" + + " <engine><proton><searchable-copies>3</searchable-copies></proton></engine>" + + " </content>" + + "</services>"; + + int numberOfHosts = 6; // We only have 6 content nodes -> 3 groups with redundancy 2 in each + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, false); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(2*3, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 + assertEquals(2*3, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 + assertEquals(2*3, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 + assertEquals("2|2|*", cluster.getRootGroup().getPartitions().get()); // Reduced from 4|4|* + assertEquals(0, cluster.getRootGroup().getNodes().size()); + assertEquals(3, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getIndex(), is("0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().size(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(0).getConfigId(), is("bar/storage/0")); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getSubgroups().get(0).getNodes().get(1).getConfigId(), is("bar/storage/1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getIndex(), is("1")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().size(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getDistributionKey(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(0).getConfigId(), is("bar/storage/2")); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getDistributionKey(), is(3)); + assertThat(cluster.getRootGroup().getSubgroups().get(1).getNodes().get(1).getConfigId(), is("bar/storage/3")); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getIndex(), is("2")); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().size(), is(2)); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getDistributionKey(), is(4)); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(0).getConfigId(), is("bar/storage/4")); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(1).getDistributionKey(), is(5)); + assertThat(cluster.getRootGroup().getSubgroups().get(2).getNodes().get(1).getConfigId(), is("bar/storage/5")); + } + + @Test + public void testUsingNodesCountAttributesAndGettingTooFewNodes() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <admin version='3.0'>" + + " <nodes count='3'/>" + // Ignored + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy reply-after='8'>12</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='24'/>" + + " <engine><proton><searchable-copies>5</searchable-copies></proton></engine>" + + " <dispatch><num-dispatch-groups>7</num-dispatch-groups></dispatch>" + + " </content>" + + "</services>"; + + int numberOfHosts = 4; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, false); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(4, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(4, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(4, cluster.redundancy().effectiveReadyCopies()); + assertEquals(4, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size()); + assertFalse(cluster.getRootGroup().getPartitions().isPresent()); + assertEquals(4, cluster.getRootGroup().getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getNodes().size(), is(4)); + assertThat(cluster.getRootGroup().getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getNodes().get(0).getConfigId(), is("bar/storage/0")); + assertThat(cluster.getRootGroup().getNodes().get(1).getDistributionKey(), is(1)); + assertThat(cluster.getRootGroup().getNodes().get(1).getConfigId(), is("bar/storage/1")); + assertThat(cluster.getRootGroup().getNodes().get(2).getDistributionKey(), is(2)); + assertThat(cluster.getRootGroup().getNodes().get(2).getConfigId(), is("bar/storage/2")); + assertThat(cluster.getRootGroup().getNodes().get(3).getDistributionKey(), is(3)); + assertThat(cluster.getRootGroup().getNodes().get(3).getConfigId(), is("bar/storage/3")); + } + + @Test + public void testUsingNodesAndGroupCountAttributesAndGettingJustOneNode() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='3.0'>" + + " <nodes count='3'/>" + // Ignored + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy reply-after='3'>4</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='24' groups='3'/>" + + " <engine><proton><searchable-copies>3</searchable-copies></proton></engine>" + + " </content>" + + "</services>"; + + int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1 + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, false); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + ContainerCluster clusterControllers = cluster.getClusterControllers(); + assertEquals(1, clusterControllers.getContainers().size()); + assertEquals("bar-controllers", clusterControllers.getName()); + assertEquals("foo0", clusterControllers.getContainers().get(0).getHostName()); + assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); // Reduced from 3*3 + assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); // Reduced from 3*4 + assertEquals(1, cluster.redundancy().effectiveReadyCopies()); // Reduced from 3*3 + assertFalse(cluster.getRootGroup().getPartitions().isPresent()); // 1 group - > flattened -> no distribution + assertEquals(1, cluster.getRootGroup().getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getNodes().get(0).getConfigId(), is("bar/storage/0")); + } + + @Test + public void testUsingNodesCountAttributesAndGettingJustOneNode() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='3.0'>" + + " <nodes count='3'/>" + // Ignored + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy reply-after='8'>12</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='24'/>" + + " <engine><proton><searchable-copies>5</searchable-copies></proton></engine>" + + " <dispatch><num-dispatch-groups>7</num-dispatch-groups></dispatch>" + + " </content>" + + "</services>"; + + int numberOfHosts = 1; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, false); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(1, cluster.redundancy().effectiveReadyCopies()); + + assertEquals(1, cluster.getSearch().getIndexed().getDispatchSpec().getGroups().size()); + assertFalse(cluster.getRootGroup().getPartitions().isPresent()); + assertEquals(1, cluster.getRootGroup().getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().size()); + assertThat(cluster.getRootGroup().getNodes().size(), is(1)); + assertThat(cluster.getRootGroup().getNodes().get(0).getDistributionKey(), is(0)); + assertThat(cluster.getRootGroup().getNodes().get(0).getConfigId(), is("bar/storage/0")); + } + + @Test + public void testRequestingSpecificFlavors() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers><nodes count='1' dedicated='true' flavor='logserver-flavor'/></logservers>" + + " <slobroks><nodes count='2' dedicated='true' flavor='slobrok-flavor'/></slobroks>" + + " </admin>" + + " <container version='1.0' id='container'>" + + " <nodes count='4' flavor='container-flavor'/>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <controllers><nodes count='2' dedicated='true' flavor='controller-foo-flavor'/></controllers>" + + " <nodes count='5' flavor='content-foo-flavor'/>" + + " </content>" + + " <content version='1.0' id='bar'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <controllers><nodes count='3' dedicated='true' flavor='controller-bar-flavor'/></controllers>" + + " <nodes count='6' flavor='content-bar-flavor'/>" + + " </content>" + + "</services>"; + + int totalHosts = 23; + Map<String, Collection<Host>> hosts = new HashMap<>(); + hosts.put("logserver-flavor", createHosts("logserver-flavor", 1).getHosts()); + hosts.put("slobrok-flavor", createHosts("slobrok-flavor", 2).getHosts()); + hosts.put("container-flavor", createHosts("container-flavor", 4).getHosts()); + hosts.put("controller-foo-flavor", createHosts("controller-foo-flavor", 2).getHosts()); + hosts.put("content-foo-flavor", createHosts("content-foo-flavor", 5).getHosts()); + hosts.put("controller-bar-flavor", createHosts("controller-bar-flavor", 3).getHosts()); + hosts.put("content-bar-flavor", createHosts("content-bar-flavor", 6).getHosts()); + VespaModel model = createModel(services, hosts, true, 0); // fails unless the right flavors+counts are requested + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(totalHosts)); + } + + @Test + public void testJDiscOnly() throws Exception { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<jdisc version='1.0'>" + + " <search/>" + + " <nodes count='3'/>" + + "</jdisc>"; + int numberOfHosts = 3; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertEquals(numberOfHosts, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(3, model.getContainerClusters().get("jdisc").getContainers().size()); + assertNotNull(model.getAdmin().getLogserver()); + assertEquals(3, model.getAdmin().getSlobroks().size()); + } + + @Test + public void testUsingHostaliasWithProvisioner() throws Exception { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>\n" + + "\n" + + "<admin version='2.0'>" + + " <adminserver hostalias='node1'/>\n"+ + "</admin>\n" + + "<jdisc id='mydisc' version='1.0'>" + + " <handler id='myHandler'>" + + " <component id='injected' />" + + " </handler>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + "</jdisc>" + + "</services>"; + int numberOfHosts = 1; + Hosts hosts = createHosts(numberOfHosts); + VespaModel model = createModel(services, hosts, true); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getAdmin().getSlobroks().size()); + } + + @Test + public void testThatStandaloneSyntaxWorksOnHostedVespa() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<jdisc id='foo' version='1.0'>" + + " <http>" + + " <server id='server1' port='" + Defaults.getDefaults().vespaWebServicePort() + "' />" + + " </http>" + + "</jdisc>"; + Hosts hosts = createHosts(1); + VespaModel model = createModel(services, hosts, true); + assertThat(model.getHosts().size(), is(1)); + assertThat(model.getContainerClusters().size(), is(1)); + } + + /** Recreate the combination used in some factory tests */ + @Test + public void testMultitenantButNotHosted() throws Exception { + String services = + "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + " </jdisc>" + + " <content id='storage' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " <node distribution-key='1' hostalias='node1'/>" + + " </group>" + + " <tuning>" + + " <cluster-controller>" + + " <transition-time>0</transition-time>" + + " </cluster-controller>" + + " </tuning>" + + " <documents>" + + " <document mode='store-only' type='type1'/>" + + " </documents>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " </content>" + + " </services>"; + + VespaModel model = createNonProvisionedMultitenantModel(services); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + ContentCluster content = model.getContentClusters().get("storage"); + assertEquals(2, content.getRootGroup().getNodes().size()); + ContainerCluster controller = content.getClusterControllers(); + assertEquals(1, controller.getContainers().size()); + } + + @Test + public void testMultitenantButNotHostedSharedContentNode() throws Exception { + String services = + "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + " </jdisc>" + + " <content id='storage' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " <node distribution-key='1' hostalias='node1'/>" + + " </group>" + + " <tuning>" + + " <cluster-controller>" + + " <transition-time>0</transition-time>" + + " </cluster-controller>" + + " </tuning>" + + " <documents>" + + " <document mode='store-only' type='type1'/>" + + " </documents>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " </content>" + + " <content id='search' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " </group>" + + " <documents>" + + " <document type='type1'/>" + + " </documents>" + + " </content>" + + " </services>"; + + VespaModel model = createNonProvisionedMultitenantModel(services); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + ContentCluster content = model.getContentClusters().get("storage"); + assertEquals(2, content.getRootGroup().getNodes().size()); + ContainerCluster controller = content.getClusterControllers(); + assertEquals(1, controller.getContainers().size()); + } + + private VespaModel createNonProvisionedMultitenantModel(String services) throws ParseException { + final VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + final ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; + DeployState deployState = new DeployState.Builder().applicationPackage(appPkg). + properties((new DeployProperties.Builder()).multitenant(true).build()). + build(); + return modelCreatorWithMockPkg.create(false, deployState); + } + + @Test + public void testThatTldConfigIdsAreDeterministic() throws ParseException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'/>" + + " <jdisc version='1.0' id='jdisc0'>" + + " <search/>" + + " <nodes count='2'/>" + + " </jdisc>" + + " <jdisc version='1.0' id='jdisc1'>" + + " <search/>" + + " <nodes count='2'/>" + + " </jdisc>" + + " <content version='1.0' id='content0'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 8; + + { + Hosts hosts = createHosts(numberOfHosts); + // Nodes used will be foo0, foo1, .. and so on. + VespaModel model = createModel(services, hosts, true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + final Map<String, ContentCluster> contentClusters = model.getContentClusters(); + assertEquals(2, contentClusters.size()); + + checkThatTldAndContainerRunningOnSameHostHaveSameId( + model.getContainerClusters().values(), + model.getContentClusters().values(), + 0); + } + + { + Hosts hosts = createHosts(numberOfHosts + 1); + // Start numbering nodes with index 1 and retire first node + // Nodes used will be foo1, foo2, .. and so on. Containers will start with index 1, not 0 as they are in the test above + VespaModel model = createModel(services, Collections.singletonMap("default", hosts.getHosts()), true, 1, "foo0"); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); + + final Map<String, ContentCluster> contentClusters = model.getContentClusters(); + assertEquals(2, contentClusters.size()); + + checkThatTldAndContainerRunningOnSameHostHaveSameId( + model.getContainerClusters().values(), + model.getContentClusters().values(), + 1); + } + } + + private void checkThatTldAndContainerRunningOnSameHostHaveSameId(Collection<ContainerCluster> containerClusters, + Collection<ContentCluster> contentClusters, + int startIndexForContainerIds) { + for (ContentCluster contentCluster : contentClusters) { + final String contentClusterName = contentCluster.getName(); + int i = 0; + for (ContainerCluster containerCluster : containerClusters) { + final String containerClusterName = containerCluster.getName(); + for (int j = 0; j < 2; j++) { + final Dispatch tld = contentCluster.getSearch().getIndexed().getTLDs().get(2 * i + j); + final Container container = containerCluster.getContainers().get(j); + final int containerConfigIdIndex = j + startIndexForContainerIds; + + assertEquals(container.getHostName(), tld.getHostname()); + assertEquals(contentClusterName + "/search/cluster." + contentClusterName + "/tlds/" + + containerClusterName + "." + containerConfigIdIndex + ".tld." + containerConfigIdIndex, + tld.getConfigId()); + assertEquals(containerClusterName + "/" + "container." + containerConfigIdIndex, + container.getConfigId()); + } + i++; + } + } + } + + private void assertIllegalModel(String services, String expectedIllegalTag) { + try { + createModel(services, createHosts(2), true); + fail("Expected that test failed"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is(expectedIllegalTag + " is not allowed when running Vespa in a hosted environment")); + } catch (ParseException e) { + fail("Test failed unexpectedly: " + e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java new file mode 100644 index 00000000000..e48baaadfd0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.*; + +import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * @author musum + */ +public class SingleNodeProvisionerTest { + @Test + public void require_basic_works() { + SingleNodeProvisioner hostProvisioner = new SingleNodeProvisioner(); + + // 4 services, 2 host aliases, mapping to 2 host. + List<String> aliases = createAliases(); + Map<String, HostSpec> map = allocate(hostProvisioner, aliases); + assertCorrectNumberOfHost(map, 1); + assertThat(map.size(), is(2)); + assertTrue(map.keySet().containsAll(aliases)); + + // 5 services, 3 host aliases, mapping to 2 host. + aliases = createAliases(Collections.singletonList("node3")); + map = allocate(hostProvisioner, aliases); + + assertCorrectNumberOfHost(map, 1); + assertThat(map.size(), is(3)); + assertTrue(map.keySet().containsAll(aliases)); + + // 5 services, 3 host aliases, mapping to 3 host. + aliases = createAliases(Collections.singletonList("node4")); + map = allocate(hostProvisioner, aliases); + assertThat(map.size(), is(3)); + assertCorrectNumberOfHost(map, 1); + assertTrue(map.keySet().containsAll(aliases)); + } + + @Test + public void require_allocate_clustermembership_works() throws IOException, SAXException { + String servicesXml = "<services version='1.0'>" + + " <admin version='3.0'>" + + " <nodes count='1' />" + + " </admin>" + + " <jdisc version='1.0'>" + + " <search />" + + " <nodes count='1' />" + + " </jdisc>" + + "</services>"; + ApplicationPackage app = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + VespaModel model = new VespaModel(app); + assertThat(model.getHosts().size(), is(1)); + } + + + private Map<String, HostSpec> allocate(HostProvisioner provisioner, List<String> aliases) { + Map<String, HostSpec> map = new LinkedHashMap<>(); + for (String alias : aliases) { + map.put(alias, provisioner.allocateHost(alias)); + } + return map; + } + + + private void assertCorrectNumberOfHost(Map<String, HostSpec> hostToServiceMap, int expectedHostCount) { + Set<String> hostSet = new HashSet<>(); + for (HostSpec host : hostToServiceMap.values()) { + hostSet.add(host.hostname()); + } + assertThat(hostSet.size(), is(expectedHostCount)); + } + + private List<String> createAliases() { + return createAliases(new ArrayList<String>()); + } + + // Admin services on node1, qrserver on node2 + additional specs + private List<String> createAliases(Collection<String> additionalAliases) { + List<String> aliases = new ArrayList<>(); + aliases.add("node1"); + aliases.add("node1"); + aliases.add("node1"); + aliases.add("node2"); + aliases.addAll(additionalAliases); + return aliases; + } + +} diff --git a/config-model/src/test/java/com/yahoo/config/model/test/MockHosts.java b/config-model/src/test/java/com/yahoo/config/model/test/MockHosts.java new file mode 100644 index 00000000000..d68e67835d2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/config/model/test/MockHosts.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.SimpleConfigProducer; + +/** + * @author tonytv + */ +public class MockHosts { + + private final MockRoot root = new MockRoot(); + private final SimpleConfigProducer<Host> hosts = new SimpleConfigProducer<>(root, "hosts"); + + public final Host host1 = new Host(hosts, "host-01.example.yahoo.com"); + public final Host host2 = new Host(hosts, "host-02.example.yahoo.com"); + public final Host host3 = new Host(hosts, "host-03.example.yahoo.com"); + +} diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java new file mode 100644 index 00000000000..1250ac45916 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.test; + +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + TODO: Document purpose + + @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a> + @author <a href="bratseth@yahoo-inc.com>Jon S Bratseth</a> +*/ +public class SDDocumentTypeTestCase extends SearchDefinitionTestCase { + + // Verify that we can register and retrieve fields. + @Test + public void testSetGet() { + SDDocumentType docType=new SDDocumentType("testdoc"); + docType.addField("Bongle",DataType.STRING); + docType.addField("nalle",DataType.INT); + + assertNotNull(docType.getField("Bongle").getName(),"Bongle"); + assertNull(docType.getField("bongle")); + + } + @Test + public void testInheritance() { + + SDDocumentType child=new SDDocumentType("child"); + Iterator<SDDocumentType> inherited=child.getInheritedTypes().iterator(); + assertTrue(inherited.hasNext()); + assertEquals(inherited.next().getDocumentName(), VespaDocumentType.NAME); + assertFalse(inherited.hasNext()); + + child.addField("childfield",DataType.INT); + SDField overridden= child.addField("overridden", DataType.STRING); + + SDDocumentType parent1=new SDDocumentType("parent1"); + SDField overridden2= parent1.addField("overridden", DataType.STRING); + parent1.addField("parent1field",DataType.STRING); + child.inherit(parent1); + + SDDocumentType parent2=new SDDocumentType("parent2"); + parent2.addField("parent2field",DataType.STRING); + child.inherit(parent2); + + SDDocumentType root=new SDDocumentType("root"); + root.addField("rootfield",DataType.STRING); + parent1.inherit(root); + parent2.inherit(root); + + inherited=child.getInheritedTypes().iterator(); + assertEquals(VespaDocumentType.NAME,inherited.next().getDocumentName()); + assertEquals(new DataTypeName("parent1"),inherited.next().getDocumentName()); + assertEquals(new DataTypeName("parent2"),inherited.next().getDocumentName()); + assertTrue(!inherited.hasNext()); + + inherited=parent1.getInheritedTypes().iterator(); + assertEquals(VespaDocumentType.NAME,inherited.next().getDocumentName()); + assertEquals(new DataTypeName("root"),inherited.next().getDocumentName()); + assertTrue(!inherited.hasNext()); + + inherited=parent2.getInheritedTypes().iterator(); + assertEquals(VespaDocumentType.NAME,inherited.next().getDocumentName()); + assertEquals(new DataTypeName("root"),inherited.next().getDocumentName()); + assertTrue(!inherited.hasNext()); + + inherited=root.getInheritedTypes().iterator(); + assertTrue(inherited.hasNext()); + assertEquals(inherited.next().getDocumentName(), VespaDocumentType.NAME); + assertFalse(inherited.hasNext()); + + + Iterator fields=child.fieldSet().iterator(); + SDField field; + + field=(SDField)fields.next(); + assertEquals("rootfield",field.getName()); + + field=(SDField)fields.next(); + assertEquals("overridden",field.getName()); + + field=(SDField)fields.next(); + assertEquals("parent1field",field.getName()); + + field=(SDField)fields.next(); + assertEquals("parent2field",field.getName()); + + field=(SDField)fields.next(); + assertEquals("childfield",field.getName()); + + // TODO: Test uninheriting + } + /* What is this?.. DocumentTypeIds aren't used for anything as far as I can see, and is now ignored by document, H\u00F9kon + public void testId() { + Search search = new Search("cocacola"); + SDDocumentType sugar = new SDDocumentType("sugar", 3, true, new DocumentTypeId(5), search); + search.addDocument(sugar); + try { + SDDocumentType color = new SDDocumentType("color", 2, true, new DocumentTypeId(5), search); + fail(); + } catch (RuntimeException re) { + } + + SDDocumentType taste = new SDDocumentType("taste", 3, true, search); + search.addDocument(taste); + try { + SDDocumentType secondtaste = new SDDocumentType("taste", 3, true, search); + fail(); + } catch (RuntimeException re) { + } + + SDDocumentType goodtaste = new SDDocumentType("goodtaste", 3, true, search); + search.addDocument(taste); + SDDocumentType badtaste = new SDDocumentType("badtaste", 3, true, search); + search.addDocument(taste); + } + */ + +} diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java new file mode 100644 index 00000000000..ec8639d1b4d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.test; + +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import org.junit.Test; + +import static org.junit.Assert.fail; + +/** + @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a> +*/ +public class SDFieldTestCase extends SearchDefinitionTestCase { + @Test + public void testIdSettingConflict() { + SDDocumentType doc=new SDDocumentType("testdoc"); + SDField one=(SDField) doc.addField("one", DataType.STRING, false, 60); + + SDField two=(SDField) doc.addField("two", DataType.STRING, false, 61); + + try { + SDField three=(SDField) doc.addField("three", DataType.STRING, false, 60); + fail("Allowed to set duplicate id"); + } + catch (IllegalArgumentException e) { + // Success + } + } + @Test + public void testSettingReservedId() { + SDDocumentType doc=new SDDocumentType("testdoc"); + try { + SDField one=(SDField) doc.addField("one", DataType.STRING, false, 127); + fail("Allowed to set reserved id"); + } + catch (IllegalArgumentException e) { + // Success + } + + try { + SDField one=(SDField) doc.addField("one", DataType.STRING, false, 100); + fail("Allowed to set reserved id"); + } + catch (IllegalArgumentException e) { + // Success + } + + try { + SDField one=(SDField) doc.addField("one", DataType.STRING, false, -1); + fail("Allowed to set reserved id"); + } + catch (IllegalArgumentException e) { + // Success + } + SDField one= doc.addField("one", DataType.STRING); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java new file mode 100644 index 00000000000..210f2d92ac0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * tests importing of document containing array type fields + * + * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ArraysTestCase extends SearchDefinitionTestCase { + + @Test + public void testArrayImporting() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/arrays.sd"); + + SDField tags = (SDField)search.getDocument().getField("tags"); + assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); + + SDField ratings = (SDField)search.getDocument().getField("ratings"); + assertTrue(ratings.getDataType() instanceof ArrayDataType); + assertEquals(DataType.INT, ((ArrayDataType)ratings.getDataType()).getNestedType()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java new file mode 100644 index 00000000000..e9d37a731e7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.document.SDField; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * tests importing of document containing array type fields and weighted set type fields, new syntax. + * + * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ArraysWeightedSetsTestCase extends SearchDefinitionTestCase { + @Test + public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); + + SDField tags = (SDField) search.getDocument().getField("tags"); + assertTrue(tags.getDataType() instanceof ArrayDataType); + assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); + + SDField ratings = (SDField) search.getDocument().getField("ratings"); + assertTrue(ratings.getDataType() instanceof ArrayDataType); + assertEquals(DataType.INT, ((CollectionDataType)ratings.getDataType()).getNestedType()); + + SDField flags = (SDField) search.getDocument().getField("flags"); + assertTrue(flags.getDataType() instanceof WeightedSetDataType); + assertEquals(DataType.STRING, ((CollectionDataType)flags.getDataType()).getNestedType()); + + SDField banners = (SDField) search.getDocument().getField("banners"); + assertTrue(banners.getDataType() instanceof WeightedSetDataType); + assertEquals(DataType.INT, ((CollectionDataType)banners.getDataType()).getNestedType()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java new file mode 100644 index 00000000000..ddffe6d6f48 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * Attribute settings + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class AttributeSettingsTestCase extends SearchDefinitionTestCase { + + @Test + public void testAttributeSettings() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + + SDField f1=(SDField) search.getDocument().getField("f1"); + assertTrue(f1.getAttributes().size() == 1); + Attribute a1 = f1.getAttributes().get(f1.getName()); + assertThat(a1.getType(), is(Attribute.Type.LONG)); + assertThat(a1.getCollectionType(), is(Attribute.CollectionType.SINGLE)); + assertTrue(a1.isHuge()); + assertFalse(a1.isFastSearch()); + assertFalse(a1.isFastAccess()); + assertFalse(a1.isRemoveIfZero()); + assertFalse(a1.isCreateIfNonExistent()); + + SDField f2=(SDField) search.getDocument().getField("f2"); + assertTrue(f2.getAttributes().size() == 1); + Attribute a2 = f2.getAttributes().get(f2.getName()); + assertThat(a2.getType(), is(Attribute.Type.LONG)); + assertThat(a2.getCollectionType(), is(Attribute.CollectionType.SINGLE)); + assertFalse(a2.isHuge()); + assertTrue(a2.isFastSearch()); + assertFalse(a2.isFastAccess()); + assertFalse(a2.isRemoveIfZero()); + assertFalse(a2.isCreateIfNonExistent()); + assertThat(f2.getAliasToName().get("f2alias"), is("f2")); + SDField f3=(SDField) search.getDocument().getField("f3"); + assertTrue(f3.getAttributes().size() == 1); + assertThat(f3.getAliasToName().get("f3alias"), is("f3")); + + Attribute a3 = f3.getAttributes().get(f3.getName()); + assertThat(a3.getType(), is(Attribute.Type.LONG)); + assertThat(a3.getCollectionType(), is(Attribute.CollectionType.SINGLE)); + assertFalse(a3.isHuge()); + assertFalse(a3.isFastSearch()); + assertFalse(a3.isFastAccess()); + assertFalse(a3.isRemoveIfZero()); + assertFalse(a3.isCreateIfNonExistent()); + + assertWeightedSet(search,"f4",true,true); + assertWeightedSet(search,"f5",true,true); + assertWeightedSet(search,"f6",true,true); + assertWeightedSet(search,"f7",true,false); + assertWeightedSet(search,"f8",true,false); + assertWeightedSet(search,"f9",false,true); + assertWeightedSet(search,"f10",false,true); + } + + private void assertWeightedSet(Search search, String name, boolean createIfNonExistent, boolean removeIfZero) { + SDField f4 = (SDField) search.getDocument().getField(name); + assertTrue(f4.getAttributes().size() == 1); + Attribute a4 = f4.getAttributes().get(f4.getName()); + assertThat(a4.getType(), is(Attribute.Type.STRING)); + assertThat(a4.getCollectionType(), is(Attribute.CollectionType.WEIGHTEDSET)); + assertFalse(a4.isHuge()); + assertFalse(a4.isFastSearch()); + assertFalse(a4.isFastAccess()); + assertThat(removeIfZero, is(a4.isRemoveIfZero())); + assertThat(createIfNonExistent, is(a4.isCreateIfNonExistent())); + } + + @Test + public void requireThatFastAccessCanBeSet() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + SDField field = (SDField) search.getDocument().getField("fast_access"); + assertTrue(field.getAttributes().size() == 1); + Attribute attr = field.getAttributes().get(field.getName()); + assertTrue(attr.isFastAccess()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java new file mode 100644 index 00000000000..26b6444ec3a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Tests comment handling + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class CommentTestCase extends SearchDefinitionTestCase { + @Test + public void testComments() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/comment.sd"); + SDField field = search.getField("a"); + assertEquals("{ input a | tokenize normalize stem:\"SHORTEST\" | summary a | index a; }", + field.getIndexingScript().toString()); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java new file mode 100644 index 00000000000..6aebe765bc4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; +import static org.junit.Assert.fail; + +import static org.junit.Assert.assertEquals; + +/** + * Created by balder on 3/10/15. + */ +public class DiversityTestCase { + @Test + public void testDiversity() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + " field b type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " match-phase {\n" + + " diversity {\n" + + " attribute: b\n" + + " min-groups: 74\n" + + " cutoff-factor: 17.3\n" + + " cutoff-strategy: strict" + + " }\n" + + " attribute: a\n" + + " max-hits: 120\n" + + " max-filter-coverage: 0.065" + + " }\n" + + " }\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.getRankProfile(s, "parent").getMatchPhaseSettings(); + RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + assertEquals("b", diversity.getAttribute()); + assertEquals(74, diversity.getMinGroups()); + assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); + assertEquals(Diversity.CutoffStrategy.strict, diversity.getCutoffStrategy()); + assertEquals(120, matchPhase.getMaxHits()); + assertEquals("a", matchPhase.getAttribute()); + assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); + } + + private static String getMessagePrefix() { + return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; + } + @Test + public void requireSingleNumericOrString() throws ParseException { + SearchBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); + + try { + builder.build(); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); + } + } + + @Test + public void requireSingle() throws ParseException { + SearchBuilder builder = getSearchBuilder("field b type array<int> { indexing: attribute }"); + + try { + builder.build(); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); + } + } + private SearchBuilder getSearchBuilder(String diversity) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + diversity + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " match-phase {\n" + + " diversity {\n" + + " attribute: b\n" + + " min-groups: 74\n" + + " }\n" + + " attribute: a\n" + + " max-hits: 120\n" + + " }\n" + + " }\n" + + "}\n"); + return builder; + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java new file mode 100644 index 00000000000..4a3119e55b7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.*; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase { + @Test + public void testDocument() throws IOException, ParseException { + + List<String> sds = new ArrayList<>(); + sds.add("src/test/examples/music.sd"); + sds.add("src/test/examples/fieldoftypedocument.sd"); + DocumentmanagerConfig.Builder value = Deriver.getDocumentManagerConfig(sds); + assertConfigFile("src/test/examples/fieldoftypedocument.cfg", + new DocumentmanagerConfig(value).toString() + "\n"); + + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "raw:" + new DocumentmanagerConfig(value).toString()); + + + DocumentType musicType = manager.getDocumentType("music"); + assertEquals(5, musicType.getFieldCount()); + + Field intField = musicType.getField("intfield"); + assertEquals(DataType.INT, intField.getDataType()); + Field stringField = musicType.getField("stringfield"); + assertEquals(DataType.STRING, stringField.getDataType()); + Field longField = musicType.getField("longfield"); + assertEquals(DataType.LONG, longField.getDataType()); + Field summaryfeatures = musicType.getField("summaryfeatures"); + assertEquals(DataType.STRING, summaryfeatures.getDataType()); + Field rankfeatures = musicType.getField("rankfeatures"); + assertEquals(DataType.STRING, rankfeatures.getDataType()); + + + DocumentType bookType = manager.getDocumentType("book"); + assertEquals(3, bookType.getFieldCount()); + + Field musicField = bookType.getField("soundtrack"); + assertSame(musicType, musicField.getDataType()); + summaryfeatures = musicType.getField("summaryfeatures"); + assertEquals(DataType.STRING, summaryfeatures.getDataType()); + rankfeatures = musicType.getField("rankfeatures"); + assertEquals(DataType.STRING, rankfeatures.getDataType()); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java new file mode 100644 index 00000000000..150a759ecf4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionTestCase { + + @Test + public void testIncorrectRef() throws IOException, ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", registry); + new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + fail("parsing should have failed"); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); + assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java new file mode 100644 index 00000000000..b6e3dc1b442 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +/** + * Tests importing a search definition with conflicting summary types + * + * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase { + @Test + public void testImportingIncorrect() throws IOException, ParseException { + try { + SearchBuilder.buildFromFile("src/test/examples/incorrectsummarytypes.sd"); + fail("processing should have failed"); + } catch (RuntimeException e) { + assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java new file mode 100644 index 00000000000..70b3b009ee6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Rank settings + * + * @author bratseth + */ +public class IndexSettingsTestCase extends SearchDefinitionTestCase { + + @Test + public void testStemmingSettings() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/indexsettings.sd"); + + SDField usingDefault=(SDField) search.getDocument().getField("usingdefault"); + assertEquals(Stemming.SHORTEST,usingDefault.getStemming(search)); + + SDField notStemmed=(SDField) search.getDocument().getField("notstemmed"); + assertEquals(Stemming.NONE,notStemmed.getStemming(search)); + + SDField allStemmed=(SDField) search.getDocument().getField("allstemmed"); + assertEquals(Stemming.SHORTEST,allStemmed.getStemming(search)); + + SDField multiStemmed=(SDField) search.getDocument().getField("multiplestems"); + assertEquals(Stemming.MULTIPLE, multiStemmed.getStemming(search)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java new file mode 100644 index 00000000000..35f823ecd30 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +/** + * Tests that indexing statements are parsed correctly. + * + * @author <a href="mailto:frodelu@yahoo-inc.com">Frode Lundgren</a> + */ +public class IndexingParsingTestCase extends SearchDefinitionTestCase { + + @Test + public void requireThatIndexingExpressionsCanBeParsed() throws Exception { + assertNotNull(SearchBuilder.buildFromFile("src/test/examples/indexing.sd")); + } + + @Test + public void requireThatParseExceptionPositionIsCorrect() throws Exception { + try { + SearchBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 5, column 57.")) { + throw e; + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java new file mode 100644 index 00000000000..250db2609e2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * tests importing of document containing array type fields + * + * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MultipleSummariesTestCase extends SearchDefinitionTestCase { + @Test + public void testArrayImporting() throws IOException, ParseException { + SearchBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java new file mode 100644 index 00000000000..f8fe979c866 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * Tests that "name" is not allowed as name for a field. + * + * And that duplicate names are not allowed. + * + * @author <a href="mailto:larschr@yahoo-inc.com">Lars Christian Jensen</a> + */ +public class NameFieldCheckTestCase extends SearchDefinitionTestCase { + + @Test + public void testNameField() throws IOException, ParseException { + try { + SearchBuilder.buildFromFile("src/test/examples/name-check.sd"); + fail("Should throw exception."); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testDuplicateNamesInSearchDifferentType() { + try { + SearchBuilder.buildFromFile("src/test/examples/duplicatenamesinsearchdifferenttype.sd"); + fail("Should throw exception."); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(e.getMessage().matches(".*Duplicate.*different type.*")); + } + } + + @Test + public void testDuplicateNamesInDoc() { + try { + SearchBuilder.buildFromFile("src/test/examples/duplicatenamesindoc.sd"); + fail("Should throw exception."); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(e.getMessage().matches(".*Duplicate.*")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java new file mode 100644 index 00000000000..c71d089e4d4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * Tests settings outside the document + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class OutsideTestCase extends SearchDefinitionTestCase { + @Test + public void testOutsideIndex() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); + + Index defaultIndex=search.getIndex("default"); + assertTrue(defaultIndex.isPrefix()); + assertEquals("default.default",defaultIndex.aliasIterator().next()); + } + @Test + public void testOutsideSummary() throws IOException, ParseException { + SearchBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java new file mode 100644 index 00000000000..09b6a8e7b8b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import static org.junit.Assert.*; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; + +/** + * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a> + * @since 5.2 + */ + +public class PredicateDataTypeTestCase { + + private String searchSd(String field) { + return "search p {\n document p {\n" + field + "}\n}\n"; + } + + private String predicateFieldSd(String index) { + return "field pf type predicate {\n" + index + "}\n"; + } + + private String arrayPredicateFieldSd(String index) { + return "field apf type array<predicate> {\n" + index + "}\n"; + } + + private String stringFieldSd(String index) { + return "field sf type string {\n" + index + "}\n"; + } + + private String attributeFieldSd(String terms) { + return "indexing: attribute\n index {\n" + terms + "}\n"; + } + + private String arityParameter(int arity) { + return "arity: " + arity + "\n"; + } + + private String lowerBoundParameter(long bound) { + return "lower-bound: " + bound + "\n"; + } + + private String upperBoundParameter(long bound) { + return "upper-bound: " + bound + "\n"; + } + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void requireThatBuilderSetsIndexParametersCorrectly() throws ParseException { + int arity = 2; + long lowerBound = -100; + long upperBound = 100; + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(arity) + + lowerBoundParameter(lowerBound) + + upperBoundParameter(upperBound)))); + + SearchBuilder sb = SearchBuilder.createFromString(sd); + for (SDField field : sb.getSearch().allFieldsList()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertEquals(true, index.getBooleanIndexDefiniton().hasArity()); + assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); + assertEquals(true, index.getBooleanIndexDefiniton().hasLowerBound()); + assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); + assertEquals(true, index.getBooleanIndexDefiniton().hasUpperBound()); + assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderHandlesLongValues() throws ParseException { + int arity = 2; + long lowerBound = -100000000000000000L; + long upperBound = 1000000000000000000L; + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(arity) + + "lower-bound: -100000000000000000L\n" + // +'L' + upperBoundParameter(upperBound)))); + + SearchBuilder sb = SearchBuilder.createFromString(sd); + for (SDField field : sb.getSearch().allFieldsList()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); + assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); + assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderHandlesMissingParameters() throws ParseException { + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(2)))); + SearchBuilder sb = SearchBuilder.createFromString(sd); + for (SDField field : sb.getSearch().allFieldsList()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertEquals(true, index.getBooleanIndexDefiniton().hasArity()); + assertEquals(false, index.getBooleanIndexDefiniton().hasLowerBound()); + assertEquals(false, index.getBooleanIndexDefiniton().hasUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderFailsIfNoArityValue() throws ParseException { + String sd = searchSd(predicateFieldSd(attributeFieldSd(""))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Missing arity value in predicate field."); + SearchBuilder.createFromString(sd); + assertTrue(false); + } + + @Test + public void requireThatBuilderFailsIfBothIndexAndAttribute() throws ParseException { + String sd = searchSd(predicateFieldSd("indexing: summary | index | attribute\nindex { arity: 2 }")); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + SearchBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfIndex() throws ParseException { + String sd = searchSd(predicateFieldSd("indexing: summary | index \nindex { arity: 2 }")); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + SearchBuilder.createFromString(sd); + } + + + @Test + public void requireThatBuilderFailsIfIllegalArityValue() throws ParseException { + String sd = searchSd(predicateFieldSd(attributeFieldSd(arityParameter(0)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid arity value in predicate field, must be greater than 1."); + SearchBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfArityParameterExistButNotPredicateField() throws ParseException { + String sd = searchSd(stringFieldSd(attributeFieldSd(arityParameter(2)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Arity parameter is used only for predicate type fields."); + SearchBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfBoundParametersExistButNotPredicateField() throws ParseException { + String sd = searchSd( + stringFieldSd( + attributeFieldSd( + lowerBoundParameter(100) + upperBoundParameter(1000)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Parameters lower-bound and upper-bound are used only for predicate type fields."); + SearchBuilder.createFromString(sd); + } + + @Test + public void requireThatArrayOfPredicateFails() throws ParseException { + String sd = searchSd( + arrayPredicateFieldSd( + attributeFieldSd( + arityParameter(1)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Collections of predicates are not allowed."); + SearchBuilder.createFromString(sd); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java new file mode 100644 index 00000000000..570c835c617 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import org.junit.Test; + +import java.io.File; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.20 + */ +public class RankProfileRegistryTest { + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; + + @Test + public void testRankProfileInheritance() { + TestRoot root = new TestDriver().buildModel(FilesApplicationPackage.fromFile(new File(TESTDIR))); + RankProfilesConfig left = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/left"); + RankProfilesConfig right = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/right"); + System.out.println(left); + assertThat(left.rankprofile().size(), is(3)); + System.out.println("\n\n"); + System.out.println(right); + assertThat(right.rankprofile().size(), is(2)); + } + + @Test(expected = IllegalArgumentException.class) + public void testRankProfileDuplicateNameIsIllegal() { + Search search = new Search("foo", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + final RankProfile barRankProfile = new RankProfile("bar", search, rankProfileRegistry); + rankProfileRegistry.addRankProfile(barRankProfile); + rankProfileRegistry.addRankProfile(barRankProfile); + } + + @Test + public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() { + Search search = new Search("foo", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + + for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { + assertNull(rankProfileRegistry.getRankProfile(search, rankProfileName).getMacros().get("foo")); + final RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); + rankProfileWithAddedMacro.addMacro("foo", true); + rankProfileRegistry.addRankProfile(rankProfileWithAddedMacro); + assertNotNull(rankProfileRegistry.getRankProfile(search, rankProfileName).getMacros().get("foo")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java new file mode 100644 index 00000000000..79ee2df03a0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests rank profiles + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class RankProfileTestCase extends SearchDefinitionTestCase { + @Test + public void testRankProfileInheritance() { + Search search = new Search("test", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType document = new SDDocumentType("test"); + SDField a = document.addField("a", DataType.STRING); + a.setRankType(RankType.IDENTITY); + document.addField("b", DataType.STRING); + search.addDocument(document); + RankProfile child = new RankProfile("child", search, rankProfileRegistry); + child.setInherited("default"); + rankProfileRegistry.addRankProfile(child); + + Iterator<RankProfile.RankSetting> i = child.rankSettingIterator(); + + RankProfile.RankSetting setting = i.next(); + assertEquals(RankType.IDENTITY, setting.getValue()); + assertEquals("a", setting.getFieldName()); + assertEquals(RankProfile.RankSetting.Type.RANKTYPE, setting.getType()); + + setting = i.next(); + assertEquals(RankType.DEFAULT, setting.getValue()); + } + + @Test + public void testTermwiseLimitAndSomeMore() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " termwise-limit:0.78\n" + + " num-threads-per-search:8\n" + + " num-search-partitions:1200\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search search = builder.getSearch(); + RankProfile rankProfile = rankProfileRegistry.getRankProfile(search, "parent"); + assertEquals(0.78, rankProfile.getTermwiseLimit(), 0.000001); + assertEquals(8, rankProfile.getNumThreadsPerSearch()); + assertEquals(1200, rankProfile.getNumSearchPartitions()); + AttributeFields attributeFields = new AttributeFields(search); + RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, attributeFields); + assertTrue(rawRankProfile.configProperties().containsKey("vespa.matching.termwise_limit")); + assertEquals("0.78", rawRankProfile.configProperties().get("vespa.matching.termwise_limit")); + assertTrue(rawRankProfile.configProperties().containsKey("vespa.matching.numthreadspersearch")); + assertEquals("8", rawRankProfile.configProperties().get("vespa.matching.numthreadspersearch")); + assertTrue(rawRankProfile.configProperties().containsKey("vespa.matching.numsearchpartitions")); + assertEquals("1200", rawRankProfile.configProperties().get("vespa.matching.numsearchpartitions")); + } + + @Test + public void requireThatConfigIsDerivedForAttributeTypeSettings() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(registry); + builder.importString("search test {\n" + + " document test { \n" + + " field a type tensor { indexing: attribute \n attribute: tensor(x[10]) }\n" + + " field b type tensor { indexing: attribute \n attribute: tensor(y{}) }\n" + + " field c type tensor { indexing: attribute }\n" + + " }\n" + + " rank-profile p1 {}\n" + + " rank-profile p2 {}\n" + + "}"); + builder.build(); + Search search = builder.getSearch(); + + assertEquals(4, registry.allRankProfiles().size()); + assertAttributeTypeSettings(registry.getRankProfile(search, "default"), search); + assertAttributeTypeSettings(registry.getRankProfile(search, "unranked"), search); + assertAttributeTypeSettings(registry.getRankProfile(search, "p1"), search); + assertAttributeTypeSettings(registry.getRankProfile(search, "p2"), search); + } + + private static void assertAttributeTypeSettings(RankProfile profile, Search search) { + RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + assertEquals("tensor(x[10])", rawProfile.configProperties().get("vespa.type.attribute.a")); + assertEquals("tensor(y{})", rawProfile.configProperties().get("vespa.type.attribute.b")); + assertFalse(rawProfile.configProperties().containsKey("vespa.type.attribute.c")); + } + + @Test + public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(registry); + builder.importString("search test {\n" + + " document test { } \n" + + " rank-profile p1 {}\n" + + " rank-profile p2 {}\n" + + "}"); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); + Search search = builder.getSearch(); + + assertEquals(4, registry.allRankProfiles().size()); + assertQueryFeatureTypeSettings(registry.getRankProfile(search, "default"), search); + assertQueryFeatureTypeSettings(registry.getRankProfile(search, "unranked"), search); + assertQueryFeatureTypeSettings(registry.getRankProfile(search, "p1"), search); + assertQueryFeatureTypeSettings(registry.getRankProfile(search, "p2"), search); + } + + private static QueryProfiles setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(tensor1)", + FieldType.fromString("tensor(x[10])", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(tensor2)", + FieldType.fromString("tensor(y{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.invalid(tensor3)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(numeric)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return new QueryProfiles(registry); + } + + private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) { + RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + assertEquals("tensor(x[10])", rawProfile.configProperties().get("vespa.type.query.tensor1")); + assertEquals("tensor(y{})", rawProfile.configProperties().get("vespa.type.query.tensor2")); + assertFalse(rawProfile.configProperties().containsKey("vespa.type.query.tensor3")); + assertFalse(rawProfile.configProperties().containsKey("vespa.type.query.numeric")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java new file mode 100644 index 00000000000..05548e62c2b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankPropertiesTestCase extends SearchDefinitionTestCase { + + @Test + public void testRankPropertyInheritance() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " first-phase {\n" + + " expression: a\n" + + " }\n" + + " rank-properties {\n" + + " query(a): 1500 \n" + + " }\n" + + " }\n" + + " rank-profile child inherits parent {\n" + + " first-phase {\n" + + " expression: a\n" + + " }\n" + + " rank-properties {\n" + + " query(a): 2000 \n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search search = builder.getSearch(); + AttributeFields attributeFields = new AttributeFields(search); + + { + // Check declared model + RankProfile parent = rankProfileRegistry.getRankProfile(search, "parent"); + assertEquals("query(a) = 1500", parent.getRankProperties().get(0).toString()); + + // Check derived model + RawRankProfile rawParent = new RawRankProfile(parent, attributeFields); + List<Map.Entry<String, Object>> parentProperties = new ArrayList<>(rawParent.configProperties().entrySet()); + assertEquals("query(a).part0=1500", parentProperties.get(0).toString()); + } + + { + // Check declared model + RankProfile parent = rankProfileRegistry.getRankProfile(search, "child"); + assertEquals("query(a) = 2000", parent.getRankProperties().get(0).toString()); + + // Check derived model + RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "child"), attributeFields); + List<Map.Entry<String, Object>> childProperties = new ArrayList<>(rawChild.configProperties().entrySet()); + assertEquals("query(a).part0=2000", childProperties.get(0).toString()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java new file mode 100644 index 00000000000..fa20e2a5eaf --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -0,0 +1,218 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.yolean.Exceptions; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase { + + @Test + public void testConstants() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " constants {\n" + + " p1: 7 \n" + + " p2: 0 \n" + + " }\n" + + " first-phase {\n" + + " expression: p2 * (1.3 + p1 )\n" + + " }\n" + + " }\n" + + " rank-profile child1 inherits parent {\n" + + " first-phase {\n" + + " expression: a + b + c \n" + + " }\n" + + " second-phase {\n" + + " expression: a + p1 + c \n" + + " }\n" + + " constants {\n" + + " a: 1.0 \n" + + " b: 2 \n" + + " c: 3.5 \n" + + " }\n" + + " }\n" + + " rank-profile child2 inherits parent {\n" + + " constants {\n" + + " p2: 2.0 \n" + + " }\n" + + " macro foo() {\n" + + " expression: p2*p1\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); + + RankProfile child1 = rankProfileRegistry.getRankProfile(s, "child1").compile(); + assertEquals("6.5", child1.getFirstPhaseRanking().getRoot().toString()); + assertEquals("11.5", child1.getSecondPhaseRanking().getRoot().toString()); + + RankProfile child2 = rankProfileRegistry.getRankProfile(s, "child2").compile(); + assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); + assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + List<Map.Entry<String, Object>> rankProperties = new ArrayList<>(new RawRankProfile(child2, new AttributeFields(s)).configProperties().entrySet()); + assertEquals("rankingExpression(foo).rankingScript.part0=14.0", rankProperties.get(0).toString()); + assertEquals("rankingExpression(firstphase).rankingScript=16.6", rankProperties.get(2).toString()); + } + + @Test + public void testNameCollision() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " constants {\n" + + " c: 7 \n" + + " }\n" + + " macro c() {\n" + + " expression: p2*p1\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + try { + rankProfileRegistry.getRankProfile(s, "test").compile(); + fail("Should have caused an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testNegativeLiteralArgument() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " macro POP_SLOW_SCORE() {\n" + + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); + profile.parseExpressions(); // TODO: Do differently + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + } + + @Test + public void testNegativeConstantArgument() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " constants {\n" + + " myValue: -9.21034037\n" + + " }\n" + + " macro POP_SLOW_SCORE() {\n" + + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); + profile.parseExpressions(); // TODO: Do differently + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.compile().getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + } + + @Test + public void testConstantDivisorInMacro() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " macro rank_default(){\n" + + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); + assertEquals("k1 + (k2 + k3) / 100000000.0", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + } + + @Test + public void test3() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " macro rank_default(){\n" + + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); + assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java new file mode 100644 index 00000000000..f15e5c06012 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { + + @Test + public void testConstants() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " constants {\n" + + " p1: 7 \n" + + " p2: 0 \n" + + " }\n" + + " first-phase {\n" + + " expression: p1 + foo\n" + + " }\n" + + " second-phase {\n" + + " expression: p2 * foo\n" + + " }\n" + + " macro inline foo() {\n" + + " expression: 3 + p1 + p2\n" + + " }\n" + + " }\n" + + " rank-profile child inherits parent {\n" + + " first-phase {\n" + + " expression: p1 + foo + baz + bar + arg(4.0)\n" + + " }\n" + + " constants {\n" + + " p2: 2.0 \n" + + " }\n" + + " macro bar() {\n" + + " expression: p2*p1\n" + + " }\n" + + " macro inline baz() {\n" + + " expression: p2+p1+boz\n" + + " }\n" + + " macro inline boz() {\n" + + " expression: 3.0\n" + + " }\n" + + " macro inline arg(a1) {\n" + + " expression: a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(); + Search s = builder.getSearch(); + + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); + assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); + List<Map.Entry<String, Object>> parentRankProperties = new ArrayList<>(new RawRankProfile(parent, new AttributeFields(s)).configProperties().entrySet()); + assertEquals("rankingExpression(foo).rankingScript.part0=10.0", parentRankProperties.get(0).toString()); + assertEquals("rankingExpression(firstphase).rankingScript=17.0", parentRankProperties.get(2).toString()); + assertEquals("rankingExpression(secondphase).rankingScript=0.0", parentRankProperties.get(4).toString()); + + RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(); + assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); + assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); + List<Map.Entry<String, Object>> childRankProperties = new ArrayList<>(new RawRankProfile(child, new AttributeFields(s)).configProperties().entrySet()); + for (Object o : childRankProperties) System.out.println(o); + assertEquals("rankingExpression(foo).rankingScript.part0=12.0", childRankProperties.get(0).toString()); + assertEquals("rankingExpression(bar).rankingScript.part1=14.0", childRankProperties.get(1).toString()); + assertEquals("rankingExpression(boz).rankingScript.part2=3.0", childRankProperties.get(2).toString()); + assertEquals("rankingExpression(baz).rankingScript.part3=9.0 + rankingExpression(boz)", childRankProperties.get(3).toString()); + assertEquals("rankingExpression(arg).rankingScript.part4=a1 * 2", childRankProperties.get(4).toString()); + assertEquals("rankingExpression(firstphase).rankingScript=31.0 + rankingExpression(bar) + rankingExpression(arg@)", censorBindingHash(childRankProperties.get(7).toString())); + assertEquals("rankingExpression(secondphase).rankingScript=24.0", childRankProperties.get(9).toString()); + } + + /** + * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of + * each macro for each binding, using hashes to name the bound variants of the macro. + * This method censors those hashes for string comparison. + */ + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + + if ( ! areInHash) + b.append(current); + + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java new file mode 100644 index 00000000000..0c7a75d5694 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankingExpressionValidationTestCase extends SearchDefinitionTestCase { + + @Test + public void testInvalidExpressionProducesException() throws ParseException { + assertFailsExpression("&/%(/%&"); + assertFailsExpression("if(a==b,b)"); + } + + private void assertFailsExpression(String expression) throws ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + Search search = importWithExpression(expression, registry); + new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + fail("No exception on incorrect ranking expression " + expression); + } catch (IllegalArgumentException e) { + // Success + // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? + assertTrue("Got unexpected error message: " + e.getCause().getMessage(), + e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + } + } + + private Search importWithExpression(String expression, RankProfileRegistry registry) throws ParseException { + SearchBuilder builder = new SearchBuilder(registry); + builder.importString("search test {" + + " document test { " + + " field a type string { " + + " indexing: index " + + " }" + + " }" + + " rank-profile default {" + + " first-phase {" + + " expression: " + expression + + " }" + + " }" + + "}"); + builder.build(); + return builder.getSearch(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java new file mode 100644 index 00000000000..4183ffe64e3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ReservedWordsAsFieldNamesTestCase extends SearchDefinitionTestCase { + + @Test + public void testIt() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/reserved_words_as_field_names.sd"); + assertNotNull(search.getDocument().getField("inline")); + assertNotNull(search.getDocument().getField("constants")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java new file mode 100755 index 00000000000..1e5db53c3e1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporarySDDocumentType; +import com.yahoo.searchdefinition.document.TemporarySDField; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static junit.framework.TestCase.assertEquals; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SDDocumentTypeOrdererTestCase { + @Test + public void testOrder() { + List<SDDocumentType> types = new ArrayList<>(); + + SDDocumentType a = new SDDocumentType("a"); + SDDocumentType b = new SDDocumentType("b"); + SDDocumentType c = new SDDocumentType("c"); + SDDocumentType d = new SDDocumentType("d"); + SDDocumentType e = new SDDocumentType("e"); + SDDocumentType f = new SDDocumentType("f"); + SDDocumentType g = new SDDocumentType("g"); + b.inherit(new TemporarySDDocumentType(new DataTypeName("a"))); + c.inherit(new TemporarySDDocumentType(new DataTypeName("b"))); + d.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); + g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); + g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); + + SDField aFieldTypeB = new TemporarySDField("atypeb", DataType.STRING); + a.addField(aFieldTypeB); + + SDField bFieldTypeC = new TemporarySDField("btypec", DataType.STRING); + b.addField(bFieldTypeC); + + SDField cFieldTypeG = new TemporarySDField("ctypeg", DataType.STRING); + c.addField(cFieldTypeG); + + SDField gFieldTypeF = new TemporarySDField("gtypef", DataType.STRING); + g.addField(gFieldTypeF); + + SDField fFieldTypeC = new TemporarySDField("ftypec", DataType.STRING); + f.addField(fFieldTypeC); + + SDField dFieldTypeE = new TemporarySDField("dtypee", DataType.STRING); + d.addField(dFieldTypeE); + + types.add(a); + types.add(b); + types.add(c); + types.add(d); + types.add(e); + types.add(f); + types.add(g); + + SDDocumentTypeOrderer app = new SDDocumentTypeOrderer(types, new BaseDeployLogger()); + app.process(); + assertEquals(7, app.processingOrder.size()); + assertEquals(a, app.processingOrder.get(0)); + assertEquals(b, app.processingOrder.get(1)); + assertEquals(c, app.processingOrder.get(2)); + assertEquals(e, app.processingOrder.get(3)); + assertEquals(d, app.processingOrder.get(4)); + assertEquals(f, app.processingOrder.get(5)); + assertEquals(g, app.processingOrder.get(6)); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java new file mode 100644 index 00000000000..774f75e059c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.io.IOUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; + +import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; +import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; +import static org.junit.Assert.assertEquals; + +public abstract class SearchDefinitionTestCase { + + public static void assertConfigFile(String filename, String cfg) throws IOException { + assertSerializedConfigFileEquals(filename, cfg); + } + + public static void assertConfigFiles(String expectedFile, String cfgFile) throws IOException { + try { + assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile)); + } catch (AssertionError e) { + throw new AssertionError(e.getMessage() + " [not equal files: >>>"+expectedFile+"<<< and >>>"+cfgFile+"<<< in assertConfigFiles]", e); + } + } + + /** + * This is to avoid having to keep those pesky array index numbers in the config format up to date + * as new entries are added and removed. + */ + public static String readAndCensorIndexes(String file) throws IOException { + StringBuilder b = new StringBuilder(); + try (BufferedReader r = IOUtils.createReader(file)) { + int character; + boolean lastWasNewline = false; + boolean inBrackets = false; + while (-1 != (character = r.read())) { + // skip empty lines + if (character == '\n') { + if (lastWasNewline) continue; + lastWasNewline = true; + } + else { + lastWasNewline = false; + } + + // skip quoted strings + if (character == '"') { + b.appendCodePoint(character); + while (-1 != (character = r.read()) && character != '"') { + b.appendCodePoint(character); + } + } + + // skip bracket content + if (character == ']') + inBrackets = false; + if (! inBrackets) + b.appendCodePoint(character); + if (character == '[') + inBrackets = true; + } + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java new file mode 100644 index 00000000000..ec4ef766745 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import java.io.IOException; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import com.yahoo.searchdefinition.parser.ParseException; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Tests that search definitions are parsed correctly and that correct line number is reported in + * error message. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase { + + @Test + public void requireThatIndexingExpressionsCanBeParsed() throws Exception { + assertNotNull(SearchBuilder.buildFromFile("src/test/examples/simple.sd")); + } + + @Test + public void requireThatParseExceptionPositionIsCorrect() throws Exception { + try { + SearchBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); + } catch (ParseException e) { + System.out.println(e.getMessage()); + if (!e.getMessage().contains("at line 5, column 36.")) { + throw e; + } + } + } + + @Test + public void requireThatParserHandlesLexicalError() throws Exception { + try { + SearchBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 7, column 27.")) { + throw e; + } + } + } + + @Test + public void requireErrorWhenJunkAfterSearchBlock() throws IOException, ParseException { + try { + SearchBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); + fail("Illegal junk at end of SD passed"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 10, column 1")) { + throw e; + } + } + } + + @Test + public void requireErrorWhenMissingClosingSearchBracket() throws IOException, ParseException { + try { + SearchBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); + fail("SD without closing bracket passed"); + } catch (ParseException e) { + if (!e.getMessage().contains("Encountered \"<EOF>\" at line 8, column 1")) { + throw e; + } + } + } + + private static class WarningCatcher extends Handler { + volatile boolean gotYqlWarning = false; + + @Override + public void publish(LogRecord record) { + if (record.getLevel() == Level.WARNING && record.getMessage().indexOf("YQL") >= 0) { + gotYqlWarning = true; + } + } + + @Override + public void flush() { + // intentionally left blank + } + + @Override + public void close() throws SecurityException { + // intentionally left blank + } + } + + + @Test + public void requireYqlCompatibilityIsTested() throws Exception { + Logger log = Logger.getLogger("DeployLogger"); + WarningCatcher w = new WarningCatcher(); + log.addHandler(w); + assertNotNull(SearchBuilder.buildFromFile("src/test/examples/simple-with-weird-name.sd")); + log.removeHandler(w); + assertTrue(w.gotYqlWarning); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java new file mode 100644 index 00000000000..bb9680665c8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -0,0 +1,180 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.searchdefinition.document.*; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.processing.MakeAliases; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Tests importing of search definitions + * + * @author bratseth + */ +public class SearchImporterTestCase extends SearchDefinitionTestCase { + + @Test + public void testSimpleImporting() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder sb = new UnprocessingSearchBuilder(rankProfileRegistry); + sb.importFile("src/test/examples/simple.sd"); + sb.build(); + Search search = sb.getSearch(); + assertEquals("simple",search.getName()); + assertTrue(search.hasDocument()); + + SDDocumentType document=search.getDocument(); + assertEquals("simple",document.getName()); + assertEquals(12,document.getFieldCount()); + + SDField field; + Attribute attribute; + + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(); + + // First field + field=(SDField) document.getField("title"); + assertEquals(DataType.STRING,field.getDataType()); + assertEquals("{ summary | index; }", + field.getIndexingScript().toString()); + assertTrue(!search.getIndex("default").isPrefix()); + assertTrue(search.getIndex("title").isPrefix()); + Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); + assertEquals("aliaz",titleAliases.next()); + assertEquals("analias.totitle",titleAliases.next()); + assertEquals("analias.todefault", + search.getIndex("default").aliasIterator().next()); + assertEquals(RankType.IDENTITY, field.getRankType()); + assertTrue(field.getAttributes().size() == 0); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + assertTrue(field.isHeader()); + + // Second field + field=(SDField) document.getField("description"); + assertEquals(RankType.ABOUT, field.getRankType()); + assertEquals(SummaryTransform.NONE, + field.getSummaryField("description").getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, + field.getSummaryField("dyndesc").getTransform()); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + assertEquals("hallo",search.getIndex("description").aliasIterator().next()); + + // Third field + field=(SDField) document.getField("chatter"); + assertEquals(RankType.ABOUT, field.getRankType()); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + + // Fourth field + field=(SDField) document.getField("category"); + assertEquals(0, field.getAttributes().size()); + assertEquals(Stemming.NONE, field.getStemming()); + assertTrue(!field.getNormalizing().doRemoveAccents()); + + // Fifth field + field=(SDField) document.getField("popularity"); + assertEquals("{ attribute; }", + field.getIndexingScript().toString()); + + // Sixth field + field=(SDField) document.getField("measurement"); + assertEquals(DataType.INT,field.getDataType()); + assertEquals(RankType.EMPTY, field.getRankType()); + assertEquals(1, field.getAttributes().size()); + + // Seventh field + field= search.getField("categories"); + assertEquals("{ input categories_src | lowercase | normalize | index; }", + field.getIndexingScript().toString()); + assertTrue(!field.isHeader()); + + // Eight field + field= search.getField("categoriesagain"); + assertEquals("{ input categoriesagain_src | lowercase | normalize | index; }", + field.getIndexingScript().toString()); + assertTrue(field.isHeader()); + + // Ninth field + field= search.getField("exactemento"); + assertEquals("{ input exactemento_src | lowercase | index | summary; }", + field.getIndexingScript().toString()); + + // Tenth field + field = search.getField("category_arr"); + assertEquals(1, field.getAttributes().size()); + attribute = field.getAttributes().get("category_arr"); + assertNotNull(attribute); + assertEquals("category_arr", attribute.getName()); + assertEquals(Attribute.Type.STRING, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + assertTrue(field.isHeader()); + + // Eleventh field + field = search.getField("measurement_arr"); + assertEquals(1, field.getAttributes().size()); + attribute = field.getAttributes().get("measurement_arr"); + assertEquals("measurement_arr", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + // Rank Profiles + RankProfile profile=rankProfileRegistry.getRankProfile(search, "default"); + assertNotNull(profile); + assertNull(profile.getInheritedName()); + assertEquals(null,profile.getDeclaredRankSetting("measurement", + RankProfile.RankSetting.Type.RANKTYPE)); + assertEquals(RankType.EMPTY, + profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); + profile=rankProfileRegistry.getRankProfile(search, "experimental"); + assertNotNull(profile); + assertEquals("default",profile.getInheritedName()); + assertEquals(RankType.IDENTITY, + profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); + + profile=rankProfileRegistry.getRankProfile(search, "other"); + assertNotNull(profile); + assertEquals("experimental",profile.getInheritedName()); + + // The extra-document field + SDField exact=search.getField("exact"); + assertNotNull("Extra field was parsed",exact); + assertEquals("exact",exact.getName()); + assertEquals(Stemming.NONE,exact.getStemming()); + assertTrue(!exact.getNormalizing().doRemoveAccents()); + assertEquals("{ input title . \" \" . input category | summary | index; }", + exact.getIndexingScript().toString()); + assertEquals(RankType.IDENTITY, exact.getRankType()); + } + + @Test + public void testDocumentImporting() throws IOException, ParseException { + try { + // Having two documents in one sd-file is illegal. + SearchBuilder.buildFromFile("src/test/examples/documents.sd"); + fail(); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testIdImporting() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/strange.sd"); + SDField idecidemyide=(SDField) search.getDocument().getField("idecidemyide"); + assertEquals(5,idecidemyide.getId(Document.SERIALIZED_VERSION)); + SDField sodoi=(SDField) search.getDocument().getField("sodoi"); + assertEquals(7,sodoi.getId(Document.SERIALIZED_VERSION)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java new file mode 100644 index 00000000000..6a0dd6e2c1b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Stemming settings test + * + * @author bratseth + */ +public class StemmingSettingTestCase extends SearchDefinitionTestCase { + + @Test + public void testStemmingSettings() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/stemmingsetting.sd"); + + SDField artist = (SDField)search.getDocument().getField("artist"); + assertEquals(Stemming.SHORTEST, artist.getStemming(search)); + + SDField title = (SDField)search.getDocument().getField("title"); + assertEquals(Stemming.NONE, title.getStemming(search)); + + SDField song = (SDField)search.getDocument().getField("song"); + assertEquals(Stemming.MULTIPLE, song.getStemming(search)); + + SDField track = (SDField)search.getDocument().getField("track"); + assertEquals(Stemming.SHORTEST, track.getStemming(search)); + + SDField backward = (SDField)search.getDocument().getField("backward"); + assertEquals(Stemming.SHORTEST, backward.getStemming(search)); + + Index defaultIndex = search.getIndex("default"); + assertEquals(Stemming.SHORTEST, defaultIndex.getStemming()); + } + + @Test + public void requireThatStemmingIsDefaultShortest() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); + assertNull(search.getField("my_str").getStemming()); + assertEquals(Stemming.SHORTEST, search.getField("my_str").getStemming(search)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java new file mode 100755 index 00000000000..0b119ccbf0d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; +import java.io.IOException; +import static org.junit.Assert.fail; + +/** + * tests importing of document containing array type fields + * + * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class StructTestCase extends SearchDefinitionTestCase { + @Test + public void testStruct() throws IOException, ParseException { + assertConfigFile("src/test/examples/structresult.cfg", + new DocumentmanagerConfig(Deriver.getDocumentManagerConfig("src/test/examples/struct.sd")).toString() + "\n"); + } + @Test + public void testBadStruct() throws IOException { + try { + SearchBuilder.buildFromFile("src/test/examples/badstruct.sd"); + fail("Should throw exception."); + } catch (ParseException e) { + //ok! + //e.printStackTrace(); + } + } + @Test + public void testStructAndDocumentWithSameNames() throws IOException, ParseException { + try { + DocumenttypesConfig.Builder dt = Deriver.getDocumentTypesConfig("src/test/examples/structanddocumentwithsamenames.sd"); + } catch (Exception e) { + fail("Should not have thrown exception " + e.toString()); + } + } + + /** + * Declaring a struct before a document will fail, no doc type to add it to. + * @throws IOException + * @throws ParseException + */ + @Test(expected = IllegalArgumentException.class) + public void testStructOutsideDocumentIllegal() throws IOException, ParseException { + SearchBuilder.buildFromFile("src/test/examples/structoutsideofdocument.sd"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java new file mode 100644 index 00000000000..98409a01432 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static junit.framework.TestCase.fail; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class UrlFieldValidationTestCase { + + @Test + public void requireThatInheritedRiseFieldsStillCanBeInConflictButDontThrowException() throws ParseException { + SearchBuilder builder = new SearchBuilder(); + builder.importString("search test {" + + " document test { " + + " field a type uri { indexing: attribute | summary }" + + " }" + + "}"); + try { + builder.build(); + fail("Should have caused an exception"); + // success + } catch (IllegalArgumentException e) { + assertEquals("Error in field 'a' in search definition 'test': uri type fields cannot be attributes", + Exceptions.toMessageString(e)); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java new file mode 100644 index 00000000000..bd7040e153b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.*; + +/** + * Superclass of tests needing file comparisons + * + * @author bratseth + */ +public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase { + + static final String tempDir = "temp/"; + static final String searchDefRoot = "src/test/derived/"; + + private static final boolean WRITE_FILES = false; + + static { + if ("true".equals(System.getProperty("sd.updatetests"))) { + /* + * Use this when you know that your code is correct, and don't want to manually check and fix the .cfg files + * in the exporting tests. + */ + setUpUpdateTest(); + } + // Or uncomment the lines you want below AND set WRITE_FILES to true + //System.setProperty("sd.updatetests", "true"); + //System.setProperty("sd.updatetestfile", "attributes"); + //System.setProperty("sd.updatetestfile", "documentmanager"); + //System.setProperty("sd.updatetestfile", "fdispatchrc"); + //System.setProperty("sd.updatetestfile", "ilscripts"); + //System.setProperty("sd.updatetestfile", "index-info"); + //System.setProperty("sd.updatetestfile", "indexschema"); + //System.setProperty("sd.updatetestfile", "juniperrc"); + //System.setProperty("sd.updatetestfile", "pan-rtx"); + //System.setProperty("sd.updatetestfile", "pan-rtx-rtlogic"); + //System.setProperty("sd.updatetestfile", "partitions"); + //System.setProperty("sd.updatetestfile", "qr-logging"); + //System.setProperty("sd.updatetestfile", "qr-searchers"); + //System.setProperty("sd.updatetestfile", "rank-profiles"); + //System.setProperty("sd.updatetestfile", "summary"); + //System.setProperty("sd.updatetestfile", "summarymap"); + //System.setProperty("sd.updatetestfile", "translogserver"); + //System.setProperty("sd.updatetestfile", "vsmsummary"); + //System.setProperty("sd.updatetestfile", "vsmfields"); + } + + private static void setUpUpdateTest() { + try { + System.out.println("Property sd.updatetests is true, updating test files from generated ones..."); + System.out.println("Enter export test file name to be updated (eg. index-info), or blank to update every file. 'q' to quit: "); + String fileName = new BufferedReader(new InputStreamReader(System.in)).readLine(); + if ("q".equals(fileName)) { + throw new IllegalArgumentException("Aborted by user"); + } + System.setProperty("sd.updatetestfile", fileName); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + protected DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException { + File toDir = new File(tempDir + dirName); + toDir.mkdirs(); + deleteContent(toDir); + + SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/"); + //SearchBuilder builder = SearchBuilder.createFromFile(searchDefDir + name + ".sd"); + return derive(dirName, searchDefinitionName, builder); + } + + protected DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException { + DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), builder.getRankProfileRegistry()); + return export(dirName, builder, config); + } + + protected DerivedConfiguration derive(String dirName, SearchBuilder builder, Search search) throws IOException { + DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + return export(dirName, builder, config); + } + + private DerivedConfiguration export(String name, SearchBuilder builder, DerivedConfiguration config) throws IOException { + String path = exportConfig(name, config); + DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); + DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); + return config; + } + + private String exportConfig(String name, DerivedConfiguration config) throws IOException { + String path = tempDir + name; + config.export(path); + return path; + } + + /** + * Derives a config from name/name.sd below the test dir and verifies that every .cfg file in name/ has a + * corresponding file with the same content in temp/name. Versions can and should be omitted from the .cfg file + * names. This will fail if the search definition dir has multiple search definitions. + * + * @param dirName the name of the directory containing the searchdef file to verify. + * @throws ParseException if the .sd file could not be parsed. + * @throws IOException if file access failed. + */ + protected DerivedConfiguration assertCorrectDeriving(String dirName) throws IOException, ParseException { + return assertCorrectDeriving(dirName, null); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException { + DerivedConfiguration derived = derive(dirName, searchDefinitionName); + assertCorrectConfigFiles(dirName); + return derived; + } + + /** + * Asserts config is correctly derived given a builder. + * This will fail if the builder contains multiple search definitions. + */ + protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException, ParseException { + builder.build(); + DerivedConfiguration derived = derive(dirName, null, builder); + assertCorrectConfigFiles(dirName); + return derived; + } + + protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, Search search, String name) throws IOException, ParseException { + DerivedConfiguration derived = derive(name, builder, search); + assertCorrectConfigFiles(name); + return derived; + } + + /** + * Assert that search is derived into the files in the directory given by name. + * + * @param name the local name of the directory containing the files to check + * @throws IOException If file access failed. + */ + protected void assertCorrectConfigFiles(String name) throws IOException { + File[] files = new File(searchDefRoot, name).listFiles(); + if (files == null) return; + for (File file : files) { + if ( ! file.getName().endsWith(".cfg")) continue; + assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName()); + } + } + + protected void assertEqualConfig(String name, String config) throws IOException { + final String expectedConfigDirName = searchDefRoot + name + "/"; + assertEqualFiles(expectedConfigDirName + config + ".cfg", + tempDir + name + "/" + config + ".cfg"); + } + + @SuppressWarnings({ "ConstantConditions" }) + public static void assertEqualFiles(String correctFileName, String checkFileName) throws IOException { + if (WRITE_FILES || "true".equals(System.getProperty("sd.updatetests"))) { + String updateFile = System.getProperty("sd.updatetestfile"); + if (WRITE_FILES || "".equals(updateFile) || correctFileName.endsWith(updateFile + ".cfg") || correctFileName.endsWith(updateFile + ".MODEL.cfg")) { + System.out.println("Copying " + checkFileName + " to " + correctFileName); + IOUtils.copy(checkFileName, correctFileName); + return; + } + } + assertConfigFiles(correctFileName, checkFileName); + } + + protected void deleteContent(File dir) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + file.delete(); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java new file mode 100755 index 00000000000..c1b1101961a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AnnotationsTestCase extends AbstractExportingTestCase { + + @Test + public void requireThatStructRegistersIfOnlyUsedByAnnotation() throws IOException, ParseException { + assertCorrectDeriving("annotationsstruct"); + } + + @Test + public void requireThatStructRegistersIfOnlyUsedAsArrayByAnnotation() throws IOException, ParseException { + assertCorrectDeriving("annotationsstructarray"); + } + + @Test + public void testSimpleAnnotationDeriving() throws IOException, ParseException { + assertCorrectDeriving("annotationssimple"); + } + + @Test + public void testAnnotationDerivingWithImplicitStruct() throws IOException, ParseException { + assertCorrectDeriving("annotationsimplicitstruct"); + } + + @Test + public void testAnnotationDerivingInheritance() throws IOException, ParseException { + assertCorrectDeriving("annotationsinheritance"); + } + + @Test + public void testAnnotationDerivingInheritance2() throws IOException, ParseException { + assertCorrectDeriving("annotationsinheritance2"); + } + + @Test + public void testSimpleReference() throws IOException, ParseException { + assertCorrectDeriving("annotationsreference"); + } + + @Test + public void testAdvancedReference() throws IOException, ParseException { + assertCorrectDeriving("annotationsreference2"); + } + + @Test + public void testAnnotationsPolymorphy() throws IOException, ParseException { + assertCorrectDeriving("annotationspolymorphy"); + } + + /** + * An annotation declared before document {} won't work, no doc type to add it to. + * @throws IOException + * @throws ParseException + */ + @Test(expected = IllegalArgumentException.class) + public void testAnnotationOutsideOfDocumment() throws IOException, ParseException { + assertCorrectDeriving("annotationsoutsideofdocument"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java new file mode 100644 index 00000000000..461dfea5472 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests array type deriving. Indexing statements over array + * types is not yet supported, so this tests document type + * configuration deriving only. Expand later. + * + * @author bratseth + */ +public class ArraysTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("arrays"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java new file mode 100644 index 00000000000..09899f4796f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests attribute deriving + * + * @author bratseth + */ +public class AttributeListTestCase extends SearchDefinitionTestCase { + @Test + public void testDeriving() throws IOException, ParseException { + + // Test attribute importing + Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); + + // Test attribute deriving + AttributeFields attributeFields = new AttributeFields(search); + Iterator attributes = attributeFields.attributeIterator(); + Attribute attribute; + attribute = (Attribute)attributes.next(); + assertEquals("popularity", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("measurement", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("smallattribute", attribute.getName()); + assertEquals(Attribute.Type.BYTE, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("access", attribute.getName()); + assertEquals(Attribute.Type.BYTE, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("category_arr", attribute.getName()); + assertEquals(Attribute.Type.STRING, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("measurement_arr", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("popsiness", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + assertTrue(!attributes.hasNext()); + + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java new file mode 100644 index 00000000000..0947671e5c7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests attribute settings + * + * @author bratseth + */ +public class AttributesTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("attributes"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java new file mode 100644 index 00000000000..92469c69fa4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Correct casing for derived attributes + * + * @author vegardh + */ +public class CasingTestCase extends SearchDefinitionTestCase { + + @Test + public void testCasing() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/casing.sd"); + assertEquals(search.getIndex("color").getName(), "color"); + assertEquals(search.getIndex("Foo").getName(), "Foo"); + assertEquals(search.getIndex("Price").getName(), "Price"); + assertEquals(search.getAttribute("artist").getName(), "artist"); + assertEquals(search.getAttribute("Drummer").getName(), "Drummer"); + assertEquals(search.getAttribute("guitarist").getName(), "guitarist"); + assertEquals(search.getAttribute("title").getName(), "title"); + assertEquals(search.getAttribute("Trumpetist").getName(), "Trumpetist"); + assertEquals(search.getAttribute("Saxophonist").getName(), "Saxophonist"); + assertEquals(search.getAttribute("TenorSaxophonist").getName(), "TenorSaxophonist"); + assertEquals(search.getAttribute("Flutist").getName(), "Flutist"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSearchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSearchTestCase.java new file mode 100644 index 00000000000..95d3dfac3f2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSearchTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving a configuration with multiple summaries + * + * @author bratseth + */ +public class CombinedAttributeAndIndexSearchTestCase extends AbstractExportingTestCase { + + @Test + public void testMultipleSummaries() throws IOException, ParseException { + assertCorrectDeriving("combinedattributeandindexsearch"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java new file mode 100644 index 00000000000..ea18fcb5266 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests deriving using the Deriver facade + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DeriverTestCase extends SearchDefinitionTestCase { + + @Test + public void testDeriveDocManager() { + DocumentTypeManager dtm = new DocumentTypeManager(new DocumentmanagerConfig( + Deriver.getDocumentManagerConfig(new ArrayList<String>() + {{ add("src/test/derived/deriver/child.sd"); + add("src/test/derived/deriver/parent.sd"); + add("src/test/derived/deriver/grandparent.sd");}}))); + assertEquals(dtm.getDocumentType("child").getField("a").getDataType(), DataType.STRING); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DocumentDeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DocumentDeriverTestCase.java new file mode 100644 index 00000000000..6c08fb88870 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DocumentDeriverTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.SDDocumentType; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Tests deriving of documentmanager + * + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Moelster Lidal</a> + */ +public class DocumentDeriverTestCase extends AbstractExportingTestCase { + @Test + public void testDocumentDeriving() { + String root = "src/test/derived/documentderiver/"; + + List<String> files = new ArrayList<>(); + files.add(root + "newsarticle.sd"); + files.add(root + "newssummary.sd"); + files.add(root + "music.sd"); + files.add(root + "mail.sd"); + files.add(root + "compression_header.sd"); + files.add(root + "compression_both.sd"); + files.add(root + "compression_body.sd"); + + File toDir = new File("temp/documentderiver/"); + toDir.mkdir(); + + SearchBuilder builder = Deriver.deriveDocuments(files, toDir.getPath()); + try { + assertEqualFiles(root + "documentmanager.cfg", toDir.getPath() + "/documentmanager.cfg"); + } catch (IOException e) { + throw new RuntimeException("Exception while comparing files", e); + } + + SDDocumentType doc = builder.getSearch("newsarticle").getDocument(); + assertNotNull(doc); + } + @Test + public void testStructTypesNotUsed() { + String root = "src/test/derived/documentderiver/"; + + List<String> files = new ArrayList<>(); + files.add(root + "sombrero.sd"); + + File toDir = new File("temp/structtypesnotused/"); + toDir.mkdir(); + + Deriver.deriveDocuments(files, toDir.getPath()); + + DocumentTypeManager dtm = new DocumentTypeManager(); + int numBuiltInTypes = dtm.getDataTypes().size(); + dtm.configure("file:" + toDir.getPath() + "/documentmanager.cfg"); + + DocumentType webDocType = dtm.getDocumentType("webdoc"); + assertNotNull(webDocType); + + assertEquals(1, webDocType.fieldSet().size()); + Field html = webDocType.getField("html"); + assertNotNull(html); + assertEquals(DataType.STRING, html.getDataType()); + + assertEquals(numBuiltInTypes + 8, dtm.getDataTypes().size()); + + { + StructDataType keyvalue = (StructDataType) dtm.getDataType("keyvalue"); + assertNotNull(keyvalue); + assertEquals(2, keyvalue.getFields().size()); + Field key = keyvalue.getField("key"); + assertNotNull(key); + assertEquals(DataType.STRING, key.getDataType()); + Field value = keyvalue.getField("value"); + assertNotNull(value); + assertEquals(DataType.STRING, value.getDataType()); + } + { + StructDataType tagvalue = (StructDataType) dtm.getDataType("tagvalue"); + assertNotNull(tagvalue); + assertEquals(2, tagvalue.getFields().size()); + Field name = tagvalue.getField("name"); + assertNotNull(name); + assertEquals(DataType.STRING, name.getDataType()); + Field attributes = tagvalue.getField("attributes"); + assertNotNull(attributes); + assertTrue(attributes.getDataType() instanceof ArrayDataType); + assertEquals(dtm.getDataType("keyvalue"), ((ArrayDataType) attributes.getDataType()).getNestedType()); + } + { + StructDataType wordform = (StructDataType) dtm.getDataType("wordform"); + assertNotNull(wordform); + assertEquals(3, wordform.getFields().size()); + Field kind = wordform.getField("kind"); + assertNotNull(kind); + assertEquals(DataType.INT, kind.getDataType()); + Field form = wordform.getField("form"); + assertNotNull(form); + assertEquals(DataType.STRING, form.getDataType()); + Field weight = wordform.getField("weight"); + assertNotNull(weight); + assertEquals(DataType.FLOAT, weight.getDataType()); + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java new file mode 100644 index 00000000000..57d62326fe3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving rank for files from search definitions + * + * @author <a href="mailto:Jon S Bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { + + @Test + public void testDeriving() throws IOException, ParseException { + Search search = new Search("test", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType doc = new SDDocumentType("test"); + search.addDocument(doc); + doc.addField(new SDField("a", DataType.STRING)); + SDField field = new SDField("b", DataType.STRING); + field.setLiteralBoost(500); + doc.addField(field); + doc.addField(new SDField("c", DataType.STRING)); + + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); + new DerivedConfiguration(search, rankProfileRegistry); + } +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java new file mode 100644 index 00000000000..06729b1e27b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ExactMatchTestCase extends AbstractExportingTestCase { + @Test + public void testExactString() throws IOException, ParseException { + assertCorrectDeriving("exactmatch"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java new file mode 100644 index 00000000000..c93e07d57c1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests exporting + * + * @author bratseth + */ +public class ExportingTestCase extends AbstractExportingTestCase { + + @Test + public void testIndexInfoLowerCase() throws IOException, ParseException { + assertCorrectDeriving("indexinfo_lowercase"); + } + + @Test + public void testPositionArray() throws IOException, ParseException { + assertCorrectDeriving("position_array"); + } + + @Test + public void testPositionAttribute() throws IOException, ParseException { + assertCorrectDeriving("position_attribute"); + } + + @Test + public void testPositionExtra() throws IOException, ParseException { + assertCorrectDeriving("position_extra"); + } + + @Test + public void testPositionNoSummary() throws IOException, ParseException { + assertCorrectDeriving("position_nosummary"); + } + + @Test + public void testPositionSummary() throws IOException, ParseException { + assertCorrectDeriving("position_summary"); + } + + @Test + public void testUriArray() throws IOException, ParseException { + assertCorrectDeriving("uri_array"); + } + + @Test + public void testUriWSet() throws IOException, ParseException { + assertCorrectDeriving("uri_wset"); + } + + @Test + public void testMusic() throws IOException, ParseException { + assertCorrectDeriving("music"); + } + + @Test + public void testComplexPhysicalExporting() throws IOException, ParseException { + assertCorrectDeriving("complex"); + } + + @Test + public void testAttributePrefetch() throws IOException, ParseException { + assertCorrectDeriving("attributeprefetch"); + } + + @Test + public void testAdvancedIL() throws IOException, ParseException { + assertCorrectDeriving("advanced"); + } + + @Test + public void testEmptyDefaultIndex() throws IOException, ParseException { + assertCorrectDeriving("emptydefault"); + } + + @Test + public void testIndexSwitches() throws IOException, ParseException { + assertCorrectDeriving("indexswitches"); + } + + @Test + public void testRankTypes() throws IOException, ParseException { + assertCorrectDeriving("ranktypes"); + } + + @Test + public void testAttributeRank() throws IOException, ParseException { + assertCorrectDeriving("attributerank"); + } + + @Test + public void testNewRank() throws IOException, ParseException { + assertCorrectDeriving("newrank"); + } + + @Test + public void testRankExpression() throws IOException, ParseException { + assertCorrectDeriving("rankexpression"); + } + + @Test + public void testMlr() throws IOException, ParseException { + assertCorrectDeriving("mlr"); + } + + @Test + public void testMusic3() throws IOException, ParseException { + assertCorrectDeriving("music3"); + } + + @Test + public void testIndexSchema() throws IOException, ParseException { + assertCorrectDeriving("indexschema"); + } + + @Test + public void testIndexinfoFieldsets() throws IOException, ParseException { + assertCorrectDeriving("indexinfo_fieldsets"); + } + + @Test + public void testStreamingJuniper() throws IOException, ParseException { + assertCorrectDeriving("streamingjuniper"); + } + + @Test + public void testPredicateAttribute() throws IOException, ParseException { + assertCorrectDeriving("predicate_attribute"); + } + + @Test + public void testTensorField() throws IOException, ParseException { + assertCorrectDeriving("tensor"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java new file mode 100644 index 00000000000..9006cfc73de --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class GeminiTestCase extends AbstractExportingTestCase { + + @Test + public void testRanking2() throws IOException, ParseException { + DerivedConfiguration c = assertCorrectDeriving("gemini2"); + RawRankProfile p = c.getRankProfileList().getRankProfile("test"); + Map<String, String> ranking = removePartKeySuffixes(p.configProperties()); + assertEquals("attribute(right)", resolve(lookup("toplevel", ranking), ranking)); + } + + private Map<String, String> removePartKeySuffixes(Map<String, Object> p) { + Map<String, String> pWithoutSuffixes = new HashMap<>(); + for (Map.Entry<String, Object> entry : p.entrySet()) + pWithoutSuffixes.put(removePartSuffix(entry.getKey()), entry.getValue().toString()); + return pWithoutSuffixes; + } + + private String removePartSuffix(String s) { + int partIndex = s.indexOf(".part"); + if (partIndex <= 0) return s; + return s.substring(0, partIndex); + } + + /** + * Recurively resolves references to other ranking expressions - rankingExpression(name) - + * and replaces the reference by the expression + */ + private String resolve(String expression, Map<String, String> ranking) { + int referenceStartIndex; + while ((referenceStartIndex = expression.indexOf("rankingExpression(")) >= 0) { + int referenceEndIndex = expression.indexOf(")", referenceStartIndex); + expression = expression.substring(0, referenceStartIndex) + + resolve(lookup(expression.substring(referenceStartIndex + "rankingExpression(".length(), referenceEndIndex), ranking), ranking) + + expression.substring(referenceEndIndex + 1); + } + return expression; + } + + private String lookup(String expressionName, Map<String, String> ranking) { + String value = ranking.get("rankingExpression(" + expressionName + ").rankingScript"); + if (value == null) { + System.out.println("Warning: No expression found for " + expressionName); + return expressionName; + } + return value; + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java new file mode 100644 index 00000000000..ca5884accd5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * Tests that documents ids are treated as they should + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class IdTestCase extends AbstractExportingTestCase { + + @Test + @SuppressWarnings({ "deprecation" }) + public void testExplicitUpperCaseIdField() { + Search search = new Search("test", null); + SDDocumentType document = new SDDocumentType("test"); + search.addDocument(document); + SDField uri = new SDField("URI", DataType.URI); + uri.parseIndexingScript("{ summary | index }"); + document.addField(uri); + + Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()); + + assertNull(document.getField("uri")); + assertNull(document.getField("Uri")); + assertNotNull(document.getField("URI")); + } + + @Test + public void testCompleteDeriving() throws Exception { + assertCorrectDeriving("id"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java new file mode 100644 index 00000000000..f1547c9b09e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "deprecation" }) +public class IndexSchemaTestCase { + + @Test + public void requireThatPrimitiveIsNotFlattened() { + assertFlat(new Field("foo", DataType.BYTE), new Field("foo", DataType.BYTE)); + assertFlat(new Field("foo", DataType.DOUBLE), new Field("foo", DataType.DOUBLE)); + assertFlat(new Field("foo", DataType.FLOAT), new Field("foo", DataType.FLOAT)); + assertFlat(new Field("foo", DataType.INT), new Field("foo", DataType.INT)); + assertFlat(new Field("foo", DataType.LONG), new Field("foo", DataType.LONG)); + assertFlat(new Field("foo", DataType.RAW), new Field("foo", DataType.RAW)); + assertFlat(new Field("foo", DataType.STRING), new Field("foo", DataType.STRING)); + assertFlat(new Field("foo", DataType.URI), new Field("foo", DataType.URI)); + assertFlat(new Field("foo", DataType.PREDICATE), new Field("foo", DataType.PREDICATE)); + } + + @Test + public void requireThatArrayOfPrimitiveIsNotFlattened() { + assertFlat(new Field("foo", DataType.getArray(DataType.BYTE)), + new Field("foo", DataType.getArray(DataType.BYTE))); + assertFlat(new Field("foo", DataType.getArray(DataType.DOUBLE)), + new Field("foo", DataType.getArray(DataType.DOUBLE))); + assertFlat(new Field("foo", DataType.getArray(DataType.FLOAT)), + new Field("foo", DataType.getArray(DataType.FLOAT))); + assertFlat(new Field("foo", DataType.getArray(DataType.INT)), + new Field("foo", DataType.getArray(DataType.INT))); + assertFlat(new Field("foo", DataType.getArray(DataType.LONG)), + new Field("foo", DataType.getArray(DataType.LONG))); + assertFlat(new Field("foo", DataType.getArray(DataType.RAW)), + new Field("foo", DataType.getArray(DataType.RAW))); + assertFlat(new Field("foo", DataType.getArray(DataType.STRING)), + new Field("foo", DataType.getArray(DataType.STRING))); + assertFlat(new Field("foo", DataType.getArray(DataType.URI)), + new Field("foo", DataType.getArray(DataType.URI))); + assertFlat(new Field("foo", DataType.getArray(DataType.PREDICATE)), + new Field("foo", DataType.getArray(DataType.PREDICATE))); + } + + @Test + public void requireThatStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.BYTE), + new Field("foo.my_double", DataType.DOUBLE), + new Field("foo.my_float", DataType.FLOAT), + new Field("foo.my_int", DataType.INT), + new Field("foo.my_long", DataType.LONG), + new Field("foo.my_raw", DataType.RAW), + new Field("foo.my_string", DataType.STRING), + new Field("foo.my_uri", DataType.URI)); + } + + @Test + public void requireThatArrayOfStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", DataType.getArray(type)), + new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), + new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), + new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), + new Field("foo.my_int", DataType.getArray(DataType.INT)), + new Field("foo.my_long", DataType.getArray(DataType.LONG)), + new Field("foo.my_raw", DataType.getArray(DataType.RAW)), + new Field("foo.my_string", DataType.getArray(DataType.STRING)), + new Field("foo.my_uri", DataType.getArray(DataType.URI))); + } + + @Test + public void requireThatArrayOfArrayOfStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", DataType.getArray(DataType.getArray(type))), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + @Test + public void requireThatStructWithArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); + type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); + type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); + type.addField(new Field("my_int", DataType.getArray(DataType.INT))); + type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); + type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); + type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); + type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), + new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), + new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), + new Field("foo.my_int", DataType.getArray(DataType.INT)), + new Field("foo.my_long", DataType.getArray(DataType.LONG)), + new Field("foo.my_raw", DataType.getArray(DataType.RAW)), + new Field("foo.my_string", DataType.getArray(DataType.STRING)), + new Field("foo.my_uri", DataType.getArray(DataType.URI))); + } + + @Test + public void requireThatStructWithArrayOfArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.getArray(DataType.BYTE)))); + type.addField(new Field("my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE)))); + type.addField(new Field("my_float", DataType.getArray(DataType.getArray(DataType.FLOAT)))); + type.addField(new Field("my_int", DataType.getArray(DataType.getArray(DataType.INT)))); + type.addField(new Field("my_long", DataType.getArray(DataType.getArray(DataType.LONG)))); + type.addField(new Field("my_raw", DataType.getArray(DataType.getArray(DataType.RAW)))); + type.addField(new Field("my_string", DataType.getArray(DataType.getArray(DataType.STRING)))); + type.addField(new Field("my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + @Test + public void requireThatArrayOfStructWithArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); + type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); + type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); + type.addField(new Field("my_int", DataType.getArray(DataType.INT))); + type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); + type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); + type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); + type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); + + assertFlat(new Field("foo", DataType.getArray(type)), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + private static void assertFlat(Field fieldToFlatten, Field... expectedFields) { + List<Field> actual = new LinkedList<>(IndexSchema.flattenField(fieldToFlatten)); + List<Field> expected = new LinkedList<>(Arrays.asList(expectedFields)); + Collections.sort(actual); + Collections.sort(expected); + for (Field field : actual) { + if (!expected.remove(field)) { + fail("Unexpected field: " + field); + } + } + assertTrue("Missing fields: " + expected, expected.isEmpty()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java new file mode 100644 index 00000000000..cb35062e59e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DataType; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.*; + +/** + * Tests inheritance + * + * @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class InheritanceTestCase extends AbstractExportingTestCase { + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void requireThatIndexedStructFieldCanBeInherited() throws IOException, ParseException { + String dir = "src/test/derived/inheritstruct/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "parent.sd"); + builder.importFile(dir + "child.sd"); + builder.build(); + derive("inheritstruct", builder, builder.getSearch("child")); + assertCorrectConfigFiles("inheritstruct"); + } + + @Test + public void requireThatInheritFromNullIsCaught() throws IOException, ParseException { + try { + assertCorrectDeriving("inheritfromnull"); + } catch (IllegalStateException e) { + assertEquals("Document type 'foo' not found.", e.getMessage()); + } + } + + @Test + public void requireThatStructTypesAreInheritedThroughDiamond() throws IOException, ParseException { + String dir = "src/test/derived/inheritdiamond/"; + List<String> files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); + File outDir = tmpDir.newFolder("out"); + for (int startIdx = 0; startIdx < files.size(); ++startIdx) { + SearchBuilder builder = new SearchBuilder(); + for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { + String fileName = files.get(fileIdx % files.size()); + builder.importFile(dir + fileName); + } + builder.build(); + DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); + DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); + DocumentmanagerConfig dc = new DocumentmanagerConfig(b); + assertEquals(17, dc.datatype().size()); + assertNotNull(structType("child.body", dc)); + DocumentmanagerConfig.Datatype.Structtype childHeader = structType("child.header", dc); + assertEquals(childHeader.field(0).name(), "foo"); + assertEquals(childHeader.field(1).name(), "bar"); + assertEquals(childHeader.field(2).name(), "baz"); + assertEquals(childHeader.field(3).name(), "cox"); + DocumentmanagerConfig.Datatype.Documenttype child = documentType("child", dc); + assertEquals(child.inherits(0).name(), "document"); + assertEquals(child.inherits(1).name(), "father"); + assertEquals(child.inherits(2).name(), "mother"); + DocumentmanagerConfig.Datatype.Documenttype mother = documentType("mother", dc); + assertEquals(mother.inherits(0).name(), "grandparent"); + assertEquals(mother.inherits(1).name(), "document"); + } + } + + private DocumentmanagerConfig.Datatype.Structtype structType(String name, DocumentmanagerConfig dc) { + for (DocumentmanagerConfig.Datatype dt : dc.datatype()) { + for (DocumentmanagerConfig.Datatype.Structtype st : dt.structtype()) { + if (name.equals(st.name())) return st; + } + } + return null; + } + + private DocumentmanagerConfig.Datatype.Documenttype documentType(String name, DocumentmanagerConfig dc) { + for (DocumentmanagerConfig.Datatype dt : dc.datatype()) { + for (DocumentmanagerConfig.Datatype.Documenttype dot : dt.documenttype()) { + if (name.equals(dot.name())) return dot; + } + } + return null; + } + + @Test + public void requireThatStructTypesAreInheritedFromParent() throws IOException, ParseException { + String dir = "src/test/derived/inheritfromparent/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "parent.sd"); + builder.importFile(dir + "child.sd"); + builder.build(); + derive("inheritfromparent", builder, builder.getSearch("child")); + assertCorrectConfigFiles("inheritfromparent"); + } + + @Test + public void requireThatStructTypesAreInheritedFromGrandParent() throws IOException, ParseException { + String dir = "src/test/derived/inheritfromgrandparent/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "grandparent.sd"); + builder.importFile(dir + "parent.sd"); + builder.importFile(dir + "child.sd"); + builder.build(); + derive("inheritfromgrandparent", builder, builder.getSearch("child")); + assertCorrectConfigFiles("inheritfromgrandparent"); + } + + @Test + public void testInheritance() throws IOException, ParseException { + try { + String dir = "src/test/derived/inheritance/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "grandparent.sd"); + builder.importFile(dir + "father.sd"); + builder.importFile(dir + "mother.sd"); + builder.importFile(dir + "child.sd"); + builder.build(); + } catch (IllegalArgumentException e) { + assertEquals( + "Inherited document 'datatype grandparent (code: -154107656)' already contains field 'overridden'. Can not override with 'overridden'.", + e.getMessage()); + } + } + + @Test + public void testIndexSettingInheritance() { + SDDocumentType parent = new SDDocumentType("parent"); + Search parentSearch = new Search("parent", null); + parentSearch.addDocument(parent); + SDField prefixed = parent.addField("prefixed", DataType.STRING); + prefixed.parseIndexingScript("{ index }"); + prefixed.addIndex(new Index("prefixed", true)); + + SDDocumentType child = new SDDocumentType("child"); + child.inherit(parent); + Search childSearch = new Search("child", null); + childSearch.addDocument(child); + + prefixed = (SDField)child.getField("prefixed"); + assertNotNull(prefixed); + assertEquals(new Index("prefixed", true), childSearch.getIndex("prefixed")); + } + + @Test + public void testFailTypesMismatch() throws IOException, ParseException { + String root = "src/test/derived/inheritancebadtypes/"; + List<String> files = new LinkedList<>(); + files.add(root + "parent.sd"); + files.add(root + "child.sd"); + File toDir = tmpDir.newFolder("to"); + try { + Deriver.deriveDocuments(files, toDir.getPath()); + fail("Import of child SD with type mismatch worked."); + } catch (RuntimeException e) { + assertTrue(e.getMessage().matches(".*already contains field 'a'.*")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java new file mode 100644 index 00000000000..b7371b66b29 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:jon@zenior.no">Jon Bratseth</a> + */ +public class IntegerAttributeToStringIndexTestCase extends AbstractExportingTestCase { + @Test + public void testIt() throws IOException, ParseException { + assertCorrectDeriving("integerattributetostringindex"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java new file mode 100644 index 00000000000..8d5c4d94939 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class LiteralBoostTestCase extends AbstractExportingTestCase { + + /** + * Tests adding of literal boost constructs + */ + @Test + public void testLiteralBoost() { + Search search=new Search("literalboost", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType document=new SDDocumentType("literalboost"); + search.addDocument(document); + SDField field1= document.addField("a", DataType.STRING); + field1.parseIndexingScript("{ index }"); + field1.setLiteralBoost(20); + RankProfile other=new RankProfile("other", search, rankProfileRegistry); + rankProfileRegistry.addRankProfile(other); + other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); + + Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); + DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry); + + // Check attribute fields + derived.getAttributeFields(); // TODO: assert content + + // Check il script addition + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", + "clear_state | guard { input a | tokenize | index a_literal; }"), + search); + + // Check index info addition + IndexInfo indexInfo=derived.getIndexInfo(); + assertTrue(indexInfo.hasCommand("a","literal-boost")); + } + + /** + * Tests adding a literal boost in a non-default rank profile only + */ + @Test + public void testNonDefaultRankLiteralBoost() { + Search search=new Search("literalboost", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType document=new SDDocumentType("literalboost"); + search.addDocument(document); + SDField field1= document.addField("a", DataType.STRING); + field1.parseIndexingScript("{ index }"); + RankProfile other=new RankProfile("other", search, rankProfileRegistry); + rankProfileRegistry.addRankProfile(other); + other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); + + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + + // Check il script addition + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", + "clear_state | guard { input a | tokenize | index a_literal; }"), + search); + + // Check index info addition + IndexInfo indexInfo=derived.getIndexInfo(); + assertTrue(indexInfo.hasCommand("a","literal-boost")); + } + + /** Tests literal boosts in two fields going to the same index */ + @Test + public void testTwoLiteralBoostFields() { + Search search=new Search("msb", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType document=new SDDocumentType("msb"); + search.addDocument(document); + SDField field1= document.addField("title", DataType.STRING); + field1.parseIndexingScript("{ summary | index }"); + field1.setLiteralBoost(20); + SDField field2= document.addField("body", DataType.STRING); + field2.parseIndexingScript("{ summary | index }"); + field2.setLiteralBoost(20); + + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); + new DerivedConfiguration(search, rankProfileRegistry); + assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", + "clear_state | guard { input body | tokenize normalize stem:\"SHORTEST\" | summary body | index body; }", + "clear_state | guard { input title | tokenize | index title_literal; }", + "clear_state | guard { input body | tokenize | index body_literal; }"), + search); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java new file mode 100644 index 00000000000..53d29bd3ca1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Tests VDS+streaming configuration deriving + * + * @author bratseth + */ +public class MailTestCase extends AbstractExportingTestCase { + + @Test + public void testMail() throws IOException, ParseException { + String dir = "src/test/derived/mail/"; + SearchBuilder sb = new SearchBuilder(); + sb.importFile(dir + "mail.sd"); + assertCorrectDeriving(sb, dir); + } + + @Test + public void testMailDocumentsonlyDeriving() { + String root = "src/test/derived/mail/"; + File toDir = new File("temp/documentderiver/"); + if (!toDir.exists()) { + toDir.mkdir(); + } + List<String> files = new ArrayList<>(); + files.add(root + "mail.sd"); + Deriver.deriveDocuments(files, toDir.getPath()); + try { + assertEqualFiles(root + "onlydoc/documentmanager.cfg", + toDir.getPath() + "/documentmanager.cfg"); + } catch (IOException e) { + throw new RuntimeException("Exception while comparing files", e); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java new file mode 100644 index 00000000000..8538944c13c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving a configuration with multiple summaries + * + * @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class MultipleSummariesTestCase extends AbstractExportingTestCase { + @Test + @Ignore + public void testMultipleSummaries() throws IOException, ParseException { + assertCorrectDeriving("multiplesummaries"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java new file mode 100644 index 00000000000..2dc96901275 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.RankType; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; +/** + * Testing stuff related to native rank type definitions + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class NativeRankTypeDefinitionsTestCase extends SearchDefinitionTestCase { + @Test + public void testTables() { + assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable"); + assertEquals(NativeTable.Type.OCCURRENCE_COUNT.getName(), "occurrenceCountTable"); + assertEquals(NativeTable.Type.PROXIMITY.getName(), "proximityTable"); + assertEquals(NativeTable.Type.REVERSE_PROXIMITY.getName(), "reverseProximityTable"); + assertEquals(NativeTable.Type.WEIGHT.getName(), "weightTable"); + } + @Test + public void testDefinitions() { + NativeRankTypeDefinitionSet defs = new NativeRankTypeDefinitionSet("default"); + + NativeRankTypeDefinition rank; + Iterator<NativeTable> tables; + + assertEquals(4, defs.types().size()); + + { + rank = defs.getRankTypeDefinition(RankType.EMPTY); + assertNotNull(rank); + assertEquals(RankType.EMPTY, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.ABOUT); + assertNotNull(rank); + assertEquals(RankType.ABOUT, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.IDENTITY); + assertNotNull(rank); + assertEquals(RankType.IDENTITY, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.TAGS); + assertNotNull(rank); + assertEquals(RankType.TAGS, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + assertEquals(RankType.ABOUT, defs.getRankTypeDefinition(RankType.DEFAULT).getType()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java new file mode 100755 index 00000000000..0d72b90a41c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class OrderIlscriptsTestCase extends AbstractExportingTestCase { + @Test + public void testOrderIlscripts() throws IOException, ParseException { + assertCorrectDeriving("orderilscripts"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java new file mode 100644 index 00000000000..77cea993131 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving of various field types + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class PrefixExactAttributeTestCase extends AbstractExportingTestCase { + @Test + public void testTypes() throws IOException, ParseException { + assertCorrectDeriving("prefixexactattribute"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java new file mode 100644 index 00000000000..222871c78b7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests a search definition with various rank profiles having different settings + * + * @author <a href="bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankProfilesTestCase extends AbstractExportingTestCase { + @Test + public void testRankProfiles() throws IOException, ParseException { + assertCorrectDeriving("rankprofiles"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java new file mode 100644 index 00000000000..211e622e7a6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:jon@zenior.no">Jon Bratseth</a> + */ +public class RankPropertiesTestCase extends AbstractExportingTestCase { + @Test + public void testRankProperties() throws IOException, ParseException { + assertCorrectDeriving("rankproperties"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java new file mode 100644 index 00000000000..084366ddcbb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SearchOrdererTestCase extends SearchDefinitionTestCase { + + private Map<String, Search> createSearchDefinitions() { + Map<String, Search> searchDefinitions = new HashMap<>(); + + Search grandParent = createSearchDefinition("grandParent", searchDefinitions); + + Search mother = createSearchDefinition("mother", searchDefinitions); + inherit(mother, grandParent); + + Search father = createSearchDefinition("father", searchDefinitions); + inherit(father, grandParent); + + Search daugther = createSearchDefinition("daughter", searchDefinitions); + inherit(daugther, father); + inherit(daugther, mother); + + Search son = createSearchDefinition("son", searchDefinitions); + inherit(son, father); + inherit(son, mother); + + Search product = createSearchDefinition("product", searchDefinitions); + + Search pc = createSearchDefinition("pc", searchDefinitions); + inherit(pc, product); + + createSearchDefinition("alone", searchDefinitions); + + return searchDefinitions; + } + + private Search createSearchDefinition(String name, Map<String, Search> searchDefinitions) { + Search search = new Search(name, null); + SDDocumentType document = new SDDocumentType(name); + search.addDocument(document); + searchDefinitions.put(search.getName(), search); + return search; + } + + private void inherit(Search inheritee, Search inherited) { + inheritee.getDocument().inherit(inherited.getDocument()); + } + + private void assertOrder(List<String> rightOrder, List<String> input) { + Map<String, Search> searchDefinitions = createSearchDefinitions(); + SearchOrderer orderer = new SearchOrderer(); + List<Search> unordered = new ArrayList<>(); + for (String anInput : input) { + Search search = searchDefinitions.get(anInput); + assertNotNull(anInput + " exists", search); + unordered.add(search); + } + List<Search> ordered = orderer.order(unordered); + List<String> names = new LinkedList<>(); + for (int i = 0; i < rightOrder.size(); i++) { + Search search = ordered.get(i); + names.add(search.getName()); + } + assertEquals(rightOrder.toString(), names.toString()); + } + + @Test + public void testPerfectOrderingIsKept() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); + } + @Test + public void testOneLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); + } + @Test + public void testMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); + } + @Test + public void testAloneIsKeptInPlaceWithMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); + } + @Test + public void testPartialMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); + } + @Test + public void testMultilevelReorderingAccrossHierarchies() { + assertOrder(Arrays.asList("alone", "grandParent", "father", "mother", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java new file mode 100644 index 00000000000..e32a80a4eb5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * Tests really simple inheriting + */ +public class SimpleInheritTestCase extends AbstractExportingTestCase { + + @Test + public void testEmptyChild() throws IOException, ParseException { + String name = "emptychild"; + final String expectedResultsDirName = "src/test/derived/" + name + "/"; + + SearchBuilder builder = new SearchBuilder(); + builder.importFile(expectedResultsDirName + "parent.sd"); + builder.importFile(expectedResultsDirName + "child.sd"); + builder.build(); + + Search search = builder.getSearch("child"); + + String toDirName = "temp/" + name; + File toDir = new File(toDirName); + toDir.mkdirs(); + deleteContent(toDir); + + DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + config.export(toDirName); + + checkDir(toDirName, expectedResultsDirName); + } + + private void checkDir(String toDirName, String expectedResultsDirName) throws IOException { + File[] files = new File(expectedResultsDirName).listFiles(); + for (File file : files) { + if (!file.getName().endsWith(".cfg")) { + continue; + } + assertEqualFiles(file.getPath(), toDirName + "/" + file.getName()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java new file mode 100644 index 00000000000..15aa8e63220 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests sort settings + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class SortingTestCase extends AbstractExportingTestCase { + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("sorting"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java new file mode 100755 index 00000000000..ee4ad4fcd8a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +/** + * Tests VSM configuration deriving for structs + * + * @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class StreamingStructTestCase extends AbstractExportingTestCase { + + @Test + public void testStreamingStruct() throws IOException, ParseException { + assertCorrectDeriving("streamingstruct"); + } + + @Test + public void testStreamingStructExplicitDefaultSummaryClass() throws IOException, ParseException { + // Tests an issue for mail in Vespa 4.1; specific overrides of default summary class + assertCorrectDeriving("streamingstructdefault"); + } + + @Test + public void testStreamingStructDocumentsonlyDeriving() throws IOException { + String root = "src/test/derived/streamingstruct/"; + String temp = "temp/documentderiver/"; + new File(temp).mkdir(); + Deriver.deriveDocuments(Arrays.asList(root + "streamingstruct.sd"), temp); + assertEqualFiles(root + "/onlydoc/documentmanager.cfg", + temp + "/documentmanager.cfg"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java new file mode 100755 index 00000000000..1f0ca253dc0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StructAnyOrderTestCase extends AbstractExportingTestCase { + @Test + public void testStructAnyOrder() throws IOException, ParseException { + assertCorrectDeriving("structanyorder"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java new file mode 100644 index 00000000000..58316111c55 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.*; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * Tests summary map extraction + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SummaryMapTestCase extends SearchDefinitionTestCase { + @Test + public void testDeriving() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); + SummaryMap summaryMap=new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); + + Iterator transforms=summaryMap.resultTransformIterator(); + FieldResultTransform transform = (FieldResultTransform)transforms.next(); + assertEquals("dyndesc", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("dynlong", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("dyndesc2", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("measurement", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("rankfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("summaryfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("popsiness", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("popularity", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("access", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + assertTrue(!transforms.hasNext()); + } + @Test + public void testPositionDeriving() throws IOException, ParseException { + Search search = new Search("store", null); + SDDocumentType document = new SDDocumentType("store"); + search.addDocument(document); + String fieldName = "location"; + SDField field = document.addField(fieldName, PositionDataType.INSTANCE); + field.parseIndexingScript("{ attribute | summary }"); + Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()); + SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); + + Iterator transforms = summaryMap.resultTransformIterator(); + + FieldResultTransform transform = (FieldResultTransform)transforms.next(); + + assertEquals(fieldName, transform.getFieldName()); + assertEquals(SummaryTransform.GEOPOS, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals(PositionDataType.getPositionSummaryFieldName(fieldName), transform.getFieldName()); + assertEquals(SummaryTransform.POSITIONS, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals(PositionDataType.getDistanceSummaryFieldName(fieldName), transform.getFieldName()); + assertEquals(SummaryTransform.DISTANCE,transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("rankfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("summaryfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); + + transform = (FieldResultTransform)transforms.next(); + assertEquals("location_zcurve", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + assertTrue(!transforms.hasNext()); + + SummarymapConfig.Builder scb = new SummarymapConfig.Builder(); + summaryMap.getConfig(scb); + SummarymapConfig c = new SummarymapConfig(scb); + + assertEquals(-1, c.defaultoutputclass()); + assertEquals(c.override().size(), 6); + + assertEquals(c.override(0).field(), fieldName); + assertEquals(c.override(0).command(), "geopos"); + assertEquals(c.override(0).arguments(), PositionDataType.getZCurveFieldName(fieldName)); + + assertEquals(c.override(1).field(), PositionDataType.getPositionSummaryFieldName(fieldName)); + assertEquals(c.override(1).command(), "positions"); + assertEquals(c.override(1).arguments(), PositionDataType.getZCurveFieldName(fieldName)); + + assertEquals(c.override(2).field(), PositionDataType.getDistanceSummaryFieldName(fieldName)); + assertEquals(c.override(2).command(), "absdist"); + assertEquals(c.override(2).arguments(), PositionDataType.getZCurveFieldName(fieldName)); + + assertEquals(c.override(3).field(), "rankfeatures"); + assertEquals(c.override(3).command(), "rankfeatures"); + assertEquals(c.override(3).arguments(), ""); + + assertEquals(c.override(4).field(), "summaryfeatures"); + assertEquals(c.override(4).command(), "summaryfeatures"); + assertEquals(c.override(4).arguments(), ""); + + assertEquals(c.override(5).field(), "location_zcurve"); + assertEquals(c.override(5).command(), "attribute"); + assertEquals(c.override(5).arguments(), "location_zcurve"); + } + + @Test + public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + try { + Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); + } catch (Exception e) { + assertTrue(e.getMessage().matches(".*equally named field.*")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java new file mode 100644 index 00000000000..ceac1186b6e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +/** + * Tests summary extraction + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SummaryTestCase extends SearchDefinitionTestCase { + @Test + public void testDeriving() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); + SummaryClass summary=new SummaryClass(search,search.getSummary("default"), new BaseDeployLogger()); + assertEquals("default",summary.getName()); + + Iterator fields=summary.fieldIterator(); + + SummaryClassField field; + + assertEquals(13, summary.getFieldCount()); + + field=(SummaryClassField)fields.next(); + assertEquals("exactemento",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("exact",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("title",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("description",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("dyndesc",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("longdesc",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("longstat",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("dynlong",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("dyndesc2",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("measurement",field.getName()); + assertEquals(SummaryClassField.Type.INTEGER,field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("rankfeatures",field.getName()); + assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("summaryfeatures",field.getName()); + assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); + + field=(SummaryClassField)fields.next(); + assertEquals("documentid",field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING,field.getType()); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java new file mode 100644 index 00000000000..0db377f43ea --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test structs for streaming with another unrelated .sd present + * + * @author arnej27959 + */ +public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { + @Test + public void testTwoStreamingStructsExporting() throws ParseException, IOException { + + String root = "src/test/derived/twostreamingstructs"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(root + "/streamingstruct.sd"); + builder.importFile(root + "/whatever.sd"); + builder.build(); + assertCorrectDeriving(builder, builder.getSearch("streamingstruct"), root); + + builder = new SearchBuilder(); + builder.importFile(root + "/streamingstruct.sd"); + builder.importFile(root + "/whatever.sd"); + builder.build(); + assertCorrectDeriving(builder, builder.getSearch("streamingstruct"), root); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java new file mode 100644 index 00000000000..315f278f942 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +/** + * Tests automatic type conversion using multifield indices + * + * @author bratseth + */ +public class TypeConversionTestCase extends SearchDefinitionTestCase { + + /** + * Tests that exact-string stuff is not spilled over to the default index + */ + @Test + public void testExactStringToStringTypeConversion() { + Search search = new Search("test", null); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); + SDDocumentType document = new SDDocumentType("test"); + search.addDocument(document); + SDField a = new SDField("a", DataType.STRING); + a.parseIndexingScript("{ index }"); + document.addField(a); + + Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + IndexInfo indexInfo = derived.getIndexInfo(); + assertFalse(indexInfo.hasCommand("default", "compact-to-term")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java new file mode 100644 index 00000000000..7908bb3a9f3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving of various field types + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class TypesTestCase extends AbstractExportingTestCase { + + @Test + public void testTypes() throws IOException, ParseException { + assertCorrectDeriving("types"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java new file mode 100644 index 00000000000..bcf5901dec2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.IndexingScript; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public abstract class AssertIndexingScript { + + public static void assertIndexing(List<String> expected, Search search) { + assertIndexing(expected, new IndexingScript(search).expressions()); + } + + public static void assertIndexing(List<String> expected, IndexingScript script) { + assertIndexing(expected, script.expressions()); + } + + public static void assertIndexing(List<String> expected, Iterable<Expression> actual) { + List<String> parsedExpected = new LinkedList<>(); + for (String str : expected) { + try { + parsedExpected.add(Expression.fromString(str).toString()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + for (Expression actualExp : actual) { + String str = actualExp.toString(); + assertTrue("Unexpected: " + str, parsedExpected.remove(str)); + } + assertTrue("Missing: " + parsedExpected.toString(), parsedExpected.isEmpty()); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java new file mode 100644 index 00000000000..e8d6a416eec --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; + +import java.io.IOException; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public abstract class AssertSearchBuilder { + + public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { + assertNotNull(SearchBuilder.buildFromFile(searchDefinitionFileName)); + } + + public static void assertBuildFails(String searchDefinitionFileName, String expectedException) + throws IOException, ParseException { + try { + SearchBuilder.buildFromFile(searchDefinitionFileName); + fail(searchDefinitionFileName); + } catch (IllegalArgumentException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributeIndexTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributeIndexTestCase.java new file mode 100644 index 00000000000..26d22908060 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributeIndexTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +/** + * Test AttributeIndex processor. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class AttributeIndexTestCase extends SearchDefinitionTestCase { + @Test + public void testAttributeIndex() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeindex.sd"); + + assertTrue(search.getField("nosettings").getAttributes().get("nosettings") != null); + + assertTrue(search.getField("specifyname").getAttributes().get("newname") != null); + + assertTrue(search.getField("specifyname2").getAttributes().get("newname2") != null); + + assertTrue(search.getField("withstaticrankname").getAttributes().get("withstaticrankname") != null); + + assertTrue(search.getField("withstaticrankname").getAttributes().get("someothername") != null); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java new file mode 100644 index 00000000000..1fa275bc505 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; +/** + * Test AttributeProperties processor. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class AttributePropertiesTestCase extends SearchDefinitionTestCase { + @Test + public void testInvalidAttributeProperties() throws IOException, ParseException { + try { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + fail("attribute property should not be set"); + } catch (RuntimeException e) { + // empty + } + } + @Test + public void testValidAttributeProperties() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java new file mode 100644 index 00000000000..2081ac6accc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +/** + * Attributes should be implicitly exact-match in some cases + * @author vegardh + * + */ +public class AttributesExactMatchTestCase extends SearchDefinitionTestCase { + @Test + public void testAttributesExactMatch() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); + assertEquals(search.getField("color").getMatching().getType(), Matching.Type.EXACT); + assertEquals(search.getField("artist").getMatching().getType(), Matching.Type.WORD); + assertEquals(search.getField("drummer").getMatching().getType(), Matching.Type.WORD); + assertEquals(search.getField("guitarist").getMatching().getType(), Matching.Type.TEXT); + assertEquals(search.getField("saxophonist_arr").getMatching().getType(), Matching.Type.WORD); + assertEquals(search.getField("flutist").getMatching().getType(), Matching.Type.TEXT); + + assertFalse(search.getField("genre").getMatching().getType().equals(Matching.Type.EXACT)); + assertFalse(search.getField("title").getMatching().getType().equals(Matching.Type.EXACT)); + assertFalse(search.getField("trumpetist").getMatching().getType().equals(Matching.Type.EXACT)); + assertFalse(search.getField("genre").getMatching().getType().equals(Matching.Type.WORD)); + assertFalse(search.getField("title").getMatching().getType().equals(Matching.Type.WORD)); + assertFalse(search.getField("trumpetist").getMatching().getType().equals(Matching.Type.WORD)); + + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java new file mode 100644 index 00000000000..a61e59e5d7f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00F8lster Lidal</a> + */ +public class BoldingTestCase extends SearchDefinitionTestCase { + @Test + public void testBoldingNonString() throws IOException, ParseException { + try { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); + } + } +} + + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java new file mode 100644 index 00000000000..03a014b6ca6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase { + + @Test + public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("rankfeatures")); + assertNotNull(docType.getField("summaryfeatures")); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(4, docType.getFieldCount()); + } + + @Test + public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("rankfeatures")); + assertNotNull(docType.getField("summaryfeatures")); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("cox")); + assertEquals(5, docType.getFieldCount()); + } + + @Test + public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("rankfeatures")); + assertNotNull(docType.getField("summaryfeatures")); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertNotNull(docType.getField("cox")); + assertEquals(6, docType.getFieldCount()); + } + + @Test + public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("rankfeatures")); + assertNotNull(docType.getField("summaryfeatures")); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertEquals(5, docType.getFieldCount()); + } + + @Test + public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("rankfeatures")); + assertNotNull(docType.getField("summaryfeatures")); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(4, docType.getFieldCount()); + } + + @Test + public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { + SearchBuilder sb = new SearchBuilder(); + sb.importFile("src/test/examples/nextgen/simple.sd"); + sb.build(); + assertNotNull(sb.getSearch()); + new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java new file mode 100644 index 00000000000..371b0bd565f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; +public class ImplicitStructTypesTestCase extends SearchDefinitionTestCase { + @Test + public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + assertStruct(docType, PositionDataType.INSTANCE); + } + @Test + public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); + assertNotNull(search); + + SDDocumentType docType = search.getDocument(); + assertNotNull(docType); + + assertField(docType, "doc_str", DataType.STRING); + assertField(docType, "doc_str_sum", DataType.STRING); + assertField(docType, "doc_uri", DataType.URI); + assertField(docType, "docsum_str", DataType.STRING); + assertField(docType, "rankfeatures", DataType.STRING); + assertField(docType, "summaryfeatures", DataType.STRING); + } + + @SuppressWarnings({ "UnusedDeclaration" }) + private static void assertStruct(SDDocumentType docType, StructDataType expectedStruct) { + // TODO: When structs are refactored from a static register to a member of the owning document types, this test + // TODO: must be changed to retrieve struct type from the provided document type. + StructDataType structType = (StructDataType) docType.getType(expectedStruct.getName()).getStruct(); + assertNotNull(structType); + for (Field expectedField : expectedStruct.getFields()) { + Field field = structType.getField(expectedField.getName()); + assertNotNull(field); + assertEquals(expectedField.getDataType(), field.getDataType()); + } + assertEquals(expectedStruct.getFieldCount(), structType.getFieldCount()); + } + + private static void assertField(SDDocumentType docType, String fieldName, DataType type) { + Field field = getSecretField(docType, fieldName); // TODO: get rid of this stupidity + assertNotNull(field); + assertEquals(type, field.getDataType()); + assertTrue(field instanceof SDField); + } + + private static Field getSecretField(SDDocumentType docType, String fieldName) { + for (Field field : docType.fieldSet()) { + if (field.getName().equals(fieldName)) { + return field; + } + } + return null; + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java new file mode 100644 index 00000000000..09f10961074 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ImplicitSummariesTestCase { + + @Test + public void requireThatSummaryFromAttributeDoesNotWarn() throws IOException, ParseException { + LogHandler log = new LogHandler(); + Logger.getLogger("").addHandler(log); + + Search search = SearchBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); + assertNotNull(search); + assertTrue(log.records.isEmpty()); + } + + private static class LogHandler extends Handler { + + final List<LogRecord> records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + if (record.getLevel() == Level.WARNING || + record.getLevel() == Level.SEVERE) + { + records.add(record); + } + } + + @Override + public void flush() { + + } + + @Override + public void close() throws SecurityException { + + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java new file mode 100644 index 00000000000..15a47d4a4af --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSummaryFieldsTestCase extends SearchDefinitionTestCase { + + @Test + public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); + assertNotNull(search); + + DocumentSummary docsum = search.getSummary("default"); + assertNotNull(docsum); + assertNotNull(docsum.getSummaryField("rankfeatures")); + assertNotNull(docsum.getSummaryField("summaryfeatures")); + assertEquals(2, docsum.getSummaryFields().size()); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java new file mode 100644 index 00000000000..48a92f40ed6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class IndexingInputsTestCase { + + @Test + public void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", + "For search 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + + "to field 'bar' which does not exist in document type " + + "'indexing_extra_field_input_extra_field'."); + } + + @Test + public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", + "For search 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + + "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit'."); + } + + @Test + public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", + "For search 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_null'."); + } + + @Test + public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", + "For search 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_self'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java new file mode 100644 index 00000000000..71d995ece36 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; + + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class IndexingOutputsTestCase { + + @Test + public void requireThatOutputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_other_field.sd", + "For search 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void requireThatOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_conflict.sd", + "For search 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + + "to assign conflicting values to field 'bar'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java new file mode 100644 index 00000000000..5a1b71db9d7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Arrays; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { + + @Test + public void testSetLanguageRewriting() { + assertIndexingScript("{ input test | set_language; }", + createField("test", DataType.STRING, "{ set_language }")); + } + + @Test + public void testSummaryRewriting() { + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary }")); + } + + @Test + public void testDynamicSummaryRewriting() { + SDField field = createField("test", DataType.STRING, "{ summary }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary dyn; }", field); + } + + @Test + public void testSummaryRewritingWithIndexing() { + assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | index }")); + } + + @Test + public void testDynamicAndStaticSummariesRewritingWithIndexing() { + SDField field = createField("test", DataType.STRING, "{ summary | index }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + field.addSummaryField(createStaticSummaryField(field, "test")); + field.addSummaryField(createStaticSummaryField(field, "other")); + field.addSummaryField(createDynamicSummaryField(field, "dyn2")); + assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary dyn | summary dyn2 | summary other | " + + "summary test | index test; }", field); + } + + @Test + public void testIntSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.INT, "{ summary | index }")); + } + + @Test + public void testStringAttributeSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.STRING, "{ summary | attribute }")); + } + + @Test + public void testMultiblockTokenize() { + SDField field = createField("test", DataType.STRING, + "{ input test | tokenize | { summary test; }; }"); + assertIndexingScript("{ input test | tokenize | { summary test; }; }", field); + } + + @Test + public void requireThatOutputDefaultsToCurrentField() { + assertIndexingScript("{ input test | attribute test; }", + createField("test", DataType.STRING, "{ attribute; }")); + assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | index test; }", + createField("test", DataType.STRING, "{ index; }")); + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary; }")); + } + + @Test + public void testTokenizeComparisonDisregardsConfig() { + assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | tokenize | index; }")); + } + + @Test + public void testDerivingFromSimple() throws Exception { + assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", + "clear_state | guard { input category | split \";\" | attribute category_arr; }", + "clear_state | guard { input category | tokenize | index category; }", + "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categories; }", + "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categoriesagain; }", + "clear_state | guard { input chatter | tokenize normalize stem:\"SHORTEST\" | index chatter; }", + "clear_state | guard { input description | tokenize normalize stem:\"SHORTEST\" | summary description | summary dyndesc | index description; }", + "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"SHORTEST\" | index exactemento | summary exactemento; }", + "clear_state | guard { input longdesc | tokenize normalize stem:\"SHORTEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", + "clear_state | guard { input measurement | attribute measurement | summary measurement; }", + "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", + "clear_state | guard { input popularity | attribute popularity; }", + "clear_state | guard { input popularity * input measurement | attribute popsiness; }", + "clear_state | guard { input smallattribute | attribute smallattribute; }", + "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", + "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), + SearchBuilder.buildFromFile("src/test/examples/simple.sd")); + } + + @Test + public void testIndexRewrite() throws Exception { + assertIndexing( + Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + + " tokenize | index title; }", + "clear_state | guard { input title_src | summary title_s; }"), + SearchBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); + } + + @Test + public void requireThatPredicateFieldsGetOptimization() { + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ attribute; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | summary test | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ summary | attribute ; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript( + "{ 2 | set_var arity | 0L | set_var lower_bound | 1023L | set_var upper_bound | " + + "{ input test | optimize_predicate | attribute test; }; }", + createPredicateField("test", DataType.PREDICATE, "{ attribute; }", 2, OptionalLong.of(0L), OptionalLong.of(1023L))); + } + + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { + assertEquals(expectedScript, + processField(unprocessedField).toString()); + } + + private static ScriptExpression processField(SDField unprocessedField) { + SDDocumentType sdoc = new SDDocumentType("test"); + sdoc.addField(unprocessedField); + Search search = new Search("test", null); + search.addDocument(sdoc); + Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()); + return unprocessedField.getIndexingScript(); + } + + private static SDField createField(String name, DataType type, String script) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + return field; + } + + private static SDField createPredicateField( + String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + Index index = new Index("foo"); + index.setBooleanIndexDefiniton(new BooleanIndexDefinition( + OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); + field.addIndex(index); + return field; + } + + private static SummaryField createDynamicSummaryField(SDField field, String name) { + return createSummaryField(field, name, true); + } + + private static SummaryField createStaticSummaryField(SDField field, String name) { + return createSummaryField(field, name, false); + } + + private static SummaryField createSummaryField(SDField field, String name, boolean dynamic) { + SummaryField summaryField = new SummaryField(name, field.getDataType()); + if (dynamic) { + summaryField.setTransform(SummaryTransform.DYNAMICTEASER); + } + summaryField.addDestination("default"); + summaryField.addSource(field.getName()); + return summaryField; + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java new file mode 100644 index 00000000000..8218d06f781 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexingValidationTestCase extends AbstractExportingTestCase { + + @Test + public void testAttributeChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_changed.sd", + "For search 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testAttributeOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_other.sd", + "For search 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testIndexChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_changed.sd", + "For search 'indexing_index_changed', field 'foo': For expression 'index foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testIndexOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_other.sd", + "For search 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testSummaryChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_changed.sd", + "For search 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + + "to assign conflicting values to field 'foo'."); + } + + @Test + public void testSummaryOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_other.sd", + "For search 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testExtraField() throws IOException, ParseException { + assertIndexing( + Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"SHORTEST\" | index my_index | summary my_index }", + "clear_state | guard { input my_input | tokenize normalize stem:\"SHORTEST\" | index my_extra | summary my_extra }"), + SearchBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); + } + + @Test + public void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", + "For search 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + + "Attempting to assign conflicting values to field 'cox'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java new file mode 100644 index 00000000000..87ad5faf97c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuilds; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class IndexingValuesTestCase { + + @Test + public void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { + assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); + } + + @Test + public void requireThatInputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_input_other_field.sd", + "For search 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + + "modifies the value of the document field 'bar'. This is no longer supported -- declare " + + "such fields outside the document."); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java new file mode 100644 index 00000000000..0b401866932 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test + public void testIntegerIndex2Attribute() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); + search.process(); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + + SDField f; + f = search.getField("s1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("s1")); + f = search.getField("s2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("s2")); + + f = search.getField("as1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("as1")); + f = search.getField("as2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("as2")); + + f = search.getField("i1"); + assertEquals(f.getAttributes().size(), 1); + assertTrue( ! f.existsIndex("i1")); + + f = search.getField("i2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue( ! f.existsIndex("i2")); + + f = search.getField("ai1"); + assertEquals(search.getField("ai1").getAttributes().size(), 1); + assertTrue( ! search.getField("ai1").existsIndex("ai1")); + f = search.getField("ai2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue( ! f.existsIndex("ai2")); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java new file mode 100644 index 00000000000..d92a0f0aa54 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import org.junit.Test; + +import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; + +public class MatchPhaseSettingsValidatorTestCase { + + private static String getMessagePrefix() { + return "In search definition 'test', rank-profile 'default': match-phase attribute 'foo' "; + } + + @Test + public void requireThatAttributeMustExists() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", + getMessagePrefix() + "does not exists"); + } + + @Test + public void requireThatAttributeMustBeNumeric() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'string'"); + } + + @Test + public void requireThatAttributeMustBeSingleValue() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'Array<int>'"); + } + + @Test + public void requireThatAttributeMustHaveFastSearch() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", + getMessagePrefix() + "must be fast-search, but it is not"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java new file mode 100644 index 00000000000..6dccdc235b9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class NGramTestCase extends SearchDefinitionTestCase { + + @Test + public void testNGram() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/ngram.sd"); + assertNotNull(search); + + SDField gram1=search.getField("gram_1"); + assertEquals(Matching.Type.GRAM,gram1.getMatching().getType()); + assertEquals(1,gram1.getMatching().getGramSize()); + + SDField gram2=search.getField("gram_2"); + assertEquals(Matching.Type.GRAM,gram2.getMatching().getType()); + assertEquals(-1,gram2.getMatching().getGramSize()); // Not set explicitly + + SDField gram3=search.getField("gram_3"); + assertEquals(Matching.Type.GRAM,gram3.getMatching().getType()); + assertEquals(3,gram3.getMatching().getGramSize()); + + assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1",gram1.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_2 | ngram 2 | index gram_2",gram2.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_3 | ngram 3 | index gram_3",gram3.getIndexingScript().iterator().next().toString()); + + assertFalse(gram1.getNormalizing().doRemoveAccents()); + assertEquals(Stemming.NONE,gram1.getStemming()); + + List<String> queryCommands=gram1.getQueryCommands(); + assertEquals(1,queryCommands.size()); + assertEquals("ngram 1",queryCommands.get(0)); + } + + @Test + public void testInvalidNGramSetting1() throws IOException, ParseException { + try { + Search search = SearchBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'",e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting2() throws IOException, ParseException { + try { + Search search = SearchBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'",e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting3() throws IOException, ParseException { + try { + Search search = SearchBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram matching is not supported with attributes, use 'index' not 'attribute' in indexing",e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java new file mode 100644 index 00000000000..45dc5b3dc78 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Test Position processor. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class PositionTestCase { + + @Test + public void requireThatPositionCanBeAttribute() throws Exception { + Search search = SearchBuilder.buildFromFile("src/test/examples/position_attribute.sd"); + assertNull(search.getAttribute("pos")); + assertNull(search.getAttribute("pos.x")); + assertNull(search.getAttribute("pos.y")); + + assertPositionAttribute(search, "pos", Attribute.CollectionType.SINGLE); + assertPositionSummary(search, "pos", false); + } + + @Test + public void requireThatPositionCanNotBeIndex() throws Exception { + try { + SearchBuilder.buildFromFile("src/test/examples/position_index.sd"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'position_index', field 'pos': Indexing of data type 'position' is not " + + "supported, replace 'index' statement with 'attribute'.", e.getMessage()); + } + } + + @Test + public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { + Search search = SearchBuilder.buildFromFile("src/test/examples/position_summary.sd"); + assertNull(search.getAttribute("pos")); + assertNull(search.getAttribute("pos.x")); + assertNull(search.getAttribute("pos.y")); + assertNull(search.getAttribute("pos.zcurve")); + + SummaryField summary = search.getSummaryField("pos"); + assertNotNull(summary); + assertEquals(2, summary.getSourceCount()); + Iterator<SummaryField.Source> it = summary.getSources().iterator(); + assertEquals("pos.x", it.next().getName()); + assertEquals("pos.y", it.next().getName()); + assertEquals(SummaryTransform.NONE, summary.getTransform()); + + assertNull(search.getSummaryField("pos_ext.distance")); + } + + @Test + public void requireThatExtraFieldCanBePositionAttribute() throws Exception { + Search search = SearchBuilder.buildFromFile("src/test/examples/position_extra.sd"); + assertNull(search.getAttribute("pos_ext")); + assertNull(search.getAttribute("pos_ext.x")); + assertNull(search.getAttribute("pos_ext.y")); + + assertPositionAttribute(search, "pos_ext", Attribute.CollectionType.SINGLE); + assertPositionSummary(search, "pos_ext", false); + } + + @Test + public void requireThatPositionArrayIsSupported() throws Exception { + Search search = SearchBuilder.buildFromFile("src/test/examples/position_array.sd"); + assertNull(search.getAttribute("pos")); + assertNull(search.getAttribute("pos.x")); + assertNull(search.getAttribute("pos.y")); + + assertPositionAttribute(search, "pos", Attribute.CollectionType.ARRAY); + assertPositionSummary(search, "pos", true); + } + + private static void assertPositionAttribute(Search search, String fieldName, Attribute.CollectionType type) { + Attribute attribute = search.getAttribute(PositionDataType.getZCurveFieldName(fieldName)); + assertNotNull(attribute); + assertTrue(attribute.isPosition()); + assertTrue(attribute.getCollectionType().equals(type)); + assertTrue(attribute.getType().equals(Attribute.Type.LONG)); + } + + private static void assertPositionSummary(Search search, String fieldName, boolean isArray) { + assertSummaryField(search, + fieldName, + PositionDataType.getZCurveFieldName(fieldName), + (isArray ? DataType.getArray(PositionDataType.INSTANCE) : PositionDataType.INSTANCE), + SummaryTransform.GEOPOS); + assertSummaryField(search, + PositionDataType.getDistanceSummaryFieldName(fieldName), + PositionDataType.getZCurveFieldName(fieldName), + DataType.INT, + SummaryTransform.DISTANCE); + assertSummaryField(search, + PositionDataType.getPositionSummaryFieldName(fieldName), + PositionDataType.getZCurveFieldName(fieldName), + DataType.getArray(DataType.STRING), + SummaryTransform.POSITIONS); + } + + private static void assertSummaryField(Search search, String fieldName, String sourceName, DataType dataType, + SummaryTransform transform) + { + SummaryField summary = search.getSummaryField(fieldName); + assertNotNull(summary); + assertEquals(1, summary.getSourceCount()); + assertEquals(sourceName, summary.getSingleSource()); + assertEquals(dataType, summary.getDataType()); + assertEquals(transform, summary.getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java new file mode 100644 index 00000000000..9778ad20374 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests for the field "rank {" shortcut + * @author vegardh + * + */ +public class RankModifierTestCase extends SearchDefinitionTestCase { + @Test + public void testLiteral() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java new file mode 100644 index 00000000000..d3ab9aaec1e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.RankProfile.RankProperty; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.fail; + +public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase { + @Test + public void testRankPropVariables() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Search search = SearchBuilder.buildFromFile("src/test/examples/rankpropvars.sd", new BaseDeployLogger(), rankProfileRegistry); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar1", "foo"); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar_2", "bar"); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvarOne23", "baz"); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "another").getRankProperties(), "$Testvar1", "1"); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "another").getRankProperties(), "$Testvar_4", "4"); + assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "another").getRankProperties(), "$testvarFour23", "234234.234"); + } + + private void assertRankPropEquals(List<RankProperty> props, String key, String val) { + for (RankProperty prop : props) { + if (prop.getName().equals(key)) { + if (prop.getValue().equals(val)) { + return; + } + } + } + fail(key+":"+val+ " not found in rank properties."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java new file mode 100644 index 00000000000..f33106b32bd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class RankingExpressionWithTensorTestCase { + + private static class SearchFixture { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Search search; + SearchFixture(String rankProfiles) throws ParseException { + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + String sdContent = "search test {\n" + + " document test {\n" + + " }\n" + + rankProfiles + + "\n" + + "}"; + builder.importString(sdContent); + builder.build(); + search = builder.getSearch(); + } + public void assertFirstPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, getRankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString()); + } + public void assertSecondPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, getRankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString()); + } + public void assertRankProperty(String expValue, String name, String rankProfile) { + List<RankProfile.RankProperty> rankPropertyList = getRankProfile(rankProfile).getRankPropertyMap().get(name); + assertEquals(1, rankPropertyList.size()); + assertEquals(expValue, rankPropertyList.get(0).getValue()); + } + public void assertMacro(String expExpression, String macroName, String rankProfile) { + assertEquals(expExpression, getRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + } + private RankProfile getRankProfile(String rankProfile) { + return rankProfileRegistry.getRankProfile(search, rankProfile).compile(); + } + } + + @Test + public void requireThatExpressionWithSingleLineTensorCanBeParsed() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum({ {x:1}:1, {x:2,y:1}:2 })\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("sum({{x:1}:1.0,{x:2,y:1}:2.0})", "my_profile"); + } + + @Test + public void requireThatExpressionWithMultiLineTensorCanBeParsed() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression {\n" + + " sum({ {x:1}:1,\n" + + " {x:2,y:1}:2 })\n" + + " }\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("sum({{x:1}:1.0,{x:2,y:1}:2.0})", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantTensorAndTypeCanBeParsed() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1, {x:2,y:1}:2 }\n" + + " type: tensor(x{},y{})\n" + + " }\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("sum(constant(my_tensor))", "my_profile"); + f.assertRankProperty("{{x:1}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value {\n" + + " { {x:1}:1,\n" + + " {x:2,y:1}:2 }\n" + + " }\n" + + " type: tensor(x{},y{})\n" + + " }\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("sum(constant(my_tensor))", "my_profile"); + f.assertRankProperty("{{x:1}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInSecondPhaseExpression() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " second-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " }\n" + + " }"); + f.assertSecondPhaseExpression("sum(constant(my_tensor))", "my_profile"); + f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInInheritedRankProfile() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile parent {\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " }\n" + + " }\n" + + " rank-profile my_profile inherits parent {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("sum(constant(my_tensor))", "my_profile"); + f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " macro my_macro() {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " first-phase {\n" + + " expression: 5.0 + my_macro\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); + f.assertMacro("sum(constant(my_tensor))", "my_macro", "my_profile"); + f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatCombinationOfConstantTensorsAndConstantValuesCanBeUsed() throws ParseException { + SearchFixture f = new SearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: my_number_1 + sum(my_tensor) + my_number_2\n" + + " }\n" + + " constants {\n" + + " my_number_1: 3.0\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " my_number_2: 5.0\n" + + " }\n" + + " }"); + f.assertFirstPhaseExpression("3.0 + sum(constant(my_tensor)) + 5.0", "my_profile"); + f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor", "constant(my_tensor).type", "my_profile"); + } + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For constant tensor 'my_tensor' in rank profile 'my_profile': Illegal tensor type spec: Failed parsing element 'x' in type spec 'tensor(x)'"); + new SearchFixture( + " rank-profile my_profile {\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " type: tensor(x)\n" + + " }\n" + + " }\n" + + " }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java new file mode 100644 index 00000000000..083627b1211 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.*; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +// TODO: WHO? +public class RankingExpressionsTestCase extends SearchDefinitionTestCase { + + @Test + public void testMacros() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry).getSearch(); + final RankProfile macrosRankProfile = rankProfileRegistry.getRankProfile(search, "macros"); + macrosRankProfile.parseExpressions(); + final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); + assertEquals(2, macros.get("titlematch$").getFormalParams().size()); + assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); + assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); + assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); + assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); + assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); + assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); + assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + + List<Map.Entry<String, Object>> rankProperties = new ArrayList<>(new RawRankProfile(macrosRankProfile, new AttributeFields(search)).configProperties().entrySet()); + assertEquals(6, rankProperties.size()); + + assertEquals("rankingExpression(titlematch$).rankingScript.part0", rankProperties.get(0).getKey()); + assertEquals("var1 * var2 + 890", rankProperties.get(0).getValue()); + + assertEquals("rankingExpression(artistmatch).rankingScript.part1", rankProperties.get(1).getKey()); + assertEquals("78 + closeness(distance)", rankProperties.get(1).getValue()); + + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getKey()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getValue()); + + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript.part3", rankProperties.get(3).getKey()); + assertEquals("7 * 8 + 890", rankProperties.get(3).getValue()); + + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript.part2", rankProperties.get(2).getKey()); + assertEquals("4 * 5 + 890", rankProperties.get(2).getValue()); + } + + @Test(expected = IllegalArgumentException.class) + public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", registry).getSearch(); + new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java new file mode 100644 index 00000000000..cab6b94fa92 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ReservedDocumentNamesTestCase extends AbstractExportingTestCase { + + @Test + public void requireThatPositionIsAReservedDocumentName() throws IOException, ParseException { + try { + assertCorrectDeriving("reserved_position"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'position': Document name 'position' is reserved.", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java new file mode 100644 index 00000000000..c5ac1ae1ed3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.searchdefinition.*; + +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { + @Test + public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); + search.process(); + try { + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + assertTrue("This should throw and never get here", false); + } catch (IllegalArgumentException e) { + assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); + } + } + @Test + public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); + search.process(); + try { + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + assertTrue("This should throw and never get here", false); + } catch (IllegalArgumentException e) { + assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); + } + } + @Test + public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); + search.process(); + try { + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(); + assertTrue("This should throw and never get here", false); + } catch (IllegalArgumentException e) { + assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); + } + } + @Test + public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { + Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); + search.process(); + BaseDeployLogger deployLogger = new BaseDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(); + assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java new file mode 100644 index 00000000000..ed29d086bb1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * @author <a href="geirst@yahoo-inc.com">Geir Storli</a> + */ +public class TensorFieldTestCase { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); + SearchBuilder.createFromString(getSd("field f1 type array<tensor> {}")); + } + + @Test + public void requireThatTensorFieldCannotBeIndexField() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field."); + SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: index }")); + } + + @Test + public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For search 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'."); + SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: attribute \n attribute: fast-search }")); + } + + @Test + public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For attribute field 'f1': Illegal tensor type spec: Failed parsing element 'invalid' in type spec 'tensor(invalid)'"); + SearchBuilder.createFromString(getSd("field f1 type tensor { indexing: attribute \n attribute: tensor(invalid) }")); + } + + private static String getSd(String field) { + return "search test {\n document test {\n" + field + "}\n}\n"; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java new file mode 100644 index 00000000000..05bf774b1d0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import org.junit.Test; +import java.io.IOException; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class DocumentModelBuilderTestCase extends SearchDefinitionTestCase { + @Test + public void testDocumentManagerSimple() throws IOException, ParseException { + DocumentModel model = createAndTestModel("src/test/configmodel/types/types.sd"); + + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile("src/test/configmodel/types/documentmanager.cfg", + new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + @Test + // This is ignored as enums in config are not testable in this way. See bug 4748050 + public void testDocumentTypesSimple() throws IOException, ParseException { + DocumentModel model = createAndTestModel("src/test/configmodel/types/types.sd"); + + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile("src/test/configmodel/types/documenttypes.cfg", + new DocumenttypesConfig(documenttypesCfg).toString()); + } + + @Test + public void testDocumentTypesWithDocumentField() throws IOException, ParseException { + SearchBuilder search = new SearchBuilder(); + search.importFile("src/test/configmodel/types/other_doc.sd"); + search.importFile("src/test/configmodel/types/type_with_doc_field.sd"); + search.build(); + DocumentModel model = search.getModel(); + + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile("src/test/configmodel/types/documenttypes_with_doc_field.cfg", + new DocumenttypesConfig(documenttypesCfg).toString()); + } + + @Test + public void testMultipleInheritanceArray() throws IOException, ParseException { + SearchBuilder search = new SearchBuilder(); + search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTData.sd"); + search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTEdge.sd"); + search.importFile("src/test/cfg/search/data/travel/searchdefinitions/TTPOI.sd"); + search.build(); + } + + private DocumentModel createAndTestModel(String sd) throws IOException, ParseException { + SearchBuilder search = SearchBuilder.createFromFile(sd); + DocumentModel model = search.getModel(); + + assertEquals(2, model.getDocumentManager().getTypes().size()); + assertNotNull(model.getDocumentManager().getDocumentType("document")); + assertNotNull(model.getDocumentManager().getDocumentType("types")); + return model; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java new file mode 100644 index 00000000000..31d45c3b75f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.test.MockRoot; +import org.junit.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @since 5.1.14 + */ +public class HostResourceTest { + + @Test + public void next_available_baseport_is_BASE_PORT_when_no_ports_have_been_reserved() { + HostResource host = newMockHostResource(); + assertThat(host.nextAvailableBaseport(1), is(HostResource.BASE_PORT)); + } + + @Test + public void next_available_baseport_is_BASE_PORT_plus_one_when_one_port_has_been_reserved() { + HostResource host = newMockHostResource(); + host.reservePort(new TestService(1), HostResource.BASE_PORT); + assertThat(host.nextAvailableBaseport(1), is(HostResource.BASE_PORT + 1)); + } + + @Test + public void no_available_baseport_when_service_requires_more_consecutive_ports_than_available() { + HostResource host = newMockHostResource(); + + for (int p = HostResource.BASE_PORT; p < HostResource.BASE_PORT + HostResource.MAX_PORTS; p += 2) { + host.reservePort(new TestService(1), p); + } + assertThat(host.nextAvailableBaseport(2), is(0)); + + try { + host.reservePort(new TestService(2), HostResource.BASE_PORT); + } catch (RuntimeException e) { + assertThat(e.getMessage(), containsString("Too many ports are reserved")); + } + } + + @Test + public void require_exception_when_no_matching_hostalias() { + TestService service = new TestService(1); + try { + service.initService(); + } catch (RuntimeException e) { + assertThat(e.getMessage(), endsWith("No host found for service 'hostresourcetest$testservice0'. " + + "The hostalias is probably missing from hosts.xml.")); + } + } + + @Test + public void port_above_vespas_port_range_can_be_reserved() { + HostResource host = newMockHostResource(); + host.allocateService(new TestService(1), HostResource.BASE_PORT + HostResource.MAX_PORTS + 1); + } + + @Test(expected = RuntimeException.class) + public void allocating_same_port_throws_exception() { + HostResource host = newMockHostResource(); + TestService service1 = new TestService(1); + TestService service2 = new TestService(1); + + host.allocateService(service1, HostResource.BASE_PORT); + host.allocateService(service2, HostResource.BASE_PORT); + } + + @Test(expected = RuntimeException.class) + public void allocating_overlapping_ports_throws_exception() { + HostResource host = newMockHostResource(); + TestService service2 = new TestService(2); + TestService service1 = new TestService(1); + + host.allocateService(service2, HostResource.BASE_PORT); + host.allocateService(service1, HostResource.BASE_PORT + 1); + } + + + private HostResource newMockHostResource() { + return new HostResource(new Host(new MockRoot())); + } + + private class TestService extends AbstractService { + private final int portCount; + + TestService(int portCount) { + super("testService"); + this.portCount = portCount; + } + + @Override + public boolean requiresWantedPort() { + return true; + } + + @Override + public int getPortCount() { return portCount; } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/InstanceResolverTest.java b/config-model/src/test/java/com/yahoo/vespa/model/InstanceResolverTest.java new file mode 100644 index 00000000000..807e0b0ae5f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/InstanceResolverTest.java @@ -0,0 +1,228 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.test.FunctionTestConfig; +import com.yahoo.test.FunctionTestConfig.*; +import com.yahoo.test.SimpletypesConfig; +import com.yahoo.config.codegen.*; +import com.yahoo.text.StringUtilities; +import org.junit.Test; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.*; + +public class InstanceResolverTest { + + @Test + public void testApplyDefToBuilder() throws Exception { + FunctionTestConfig.Builder builder = createVariableAccessBuilder(); + InnerCNode targetDef = getDef(FunctionTestConfig.CONFIG_DEF_SCHEMA); + + // Mutate the def, user has set different schema ... + ((LeafCNode) targetDef.getChild("stringwithdef")).setDefaultValue(new DefaultValue("newDef", new DefLine.Type("string"))); + ((LeafCNode) targetDef.getChild("string_val")).setDefaultValue(new DefaultValue("newDefVal", new DefLine.Type("string"))); + ((LeafCNode) targetDef.getChild("enumwithdef")).setDefaultValue(new DefaultValue(Enumwithdef.FOOBAR2.toString(), new DefLine.Type("enum"))); + ((LeafCNode) targetDef.getChild("enum_val")).setDefaultValue(new DefaultValue(Enum_val.FOO.toString(), new DefLine.Type("enum"))); + ((LeafCNode) targetDef.getChild("basicStruct").getChild("foo")).setDefaultValue(new DefaultValue("basicSchmasic", new DefLine.Type("string"))); + ((LeafCNode) targetDef.getChild("basicStruct").getChild("bar")).setDefaultValue(new DefaultValue("89", new DefLine.Type("int"))); + InnerCNode rootStruct = ((InnerCNode) targetDef.getChild("rootStruct")); + InnerCNode innerArr = (InnerCNode) rootStruct.getChild("innerArr"); + ((LeafCNode) innerArr.getChild("boolVal")).setDefaultValue(new DefaultValue("true", new DefLine.Type("bool"))); + ((LeafCNode) innerArr.getChild("stringVal")).setDefaultValue(new DefaultValue("derp", new DefLine.Type("string"))); + InnerCNode myArray = ((InnerCNode) targetDef.getChild("myarray")); + myArray.children().put("intval", LeafCNode.newInstance(new DefLine.Type("int"), myArray, "intval", "-123424")); + targetDef.children().put("myarray", myArray); + InstanceResolver.applyDef(builder, targetDef); + FunctionTestConfig c = new FunctionTestConfig(builder); + assertEquals(c.string_val(), "foo"); + assertEquals(c.stringwithdef(), "newDef"); + assertEquals(c.enumwithdef(), Enumwithdef.FOOBAR2); + assertEquals(c.enum_val(), Enum_val.FOOBAR); + assertEquals(c.double_with_def(), -12, 0.0001); + assertEquals(c.basicStruct().foo(), "basicSchmasic"); + assertEquals(c.basicStruct().bar(), 3); + assertTrue(c.rootStruct().innerArr(0).boolVal()); + assertEquals(c.rootStruct().innerArr(0).stringVal(), "deep"); + assertEquals(c.myarray(0).intval(), -123424); + } + + /** + * Values unset on builder, trying to set them from def file, but type mismatches there + * @throws Exception + */ + @Test + public void testApplyDefToBuilderMismatches() throws Exception { + FunctionTestConfig.Builder builder = createVariableAccessBuilderManyUnset(); + InnerCNode targetDef = getDef(FunctionTestConfig.CONFIG_DEF_SCHEMA); + + // Break the defs for these, they are unset on builder: + targetDef.children().put("stringwithdef", LeafCNode.newInstance(new DefLine.Type("int"), targetDef, "stringwithdef", "1")); + targetDef.children().put("int_val", LeafCNode.newInstance(new DefLine.Type("string"), targetDef, "int_val", "fooOO")); + + InstanceResolver.applyDef(builder, targetDef); + try { + FunctionTestConfig c = new FunctionTestConfig(builder); + fail("No exception on incomplete builder"); + } catch (Exception e) { + } + } + + // copied from FunctionTest + private FunctionTestConfig.Builder createVariableAccessBuilder() { + return new FunctionTestConfig.Builder(). + bool_val(false). + bool_with_def(true). + int_val(5). + int_with_def(-14). + long_val(12345678901L). + long_with_def(-9876543210L). + double_val(41.23). + double_with_def(-12). + string_val("foo"). + //stringwithdef("bar"). + enum_val(Enum_val.FOOBAR). + //enumwithdef(Enumwithdef.BAR2). + refval(":parent:"). + refwithdef(":parent:"). + fileVal("etc"). + boolarr(false). + longarr(9223372036854775807L). + longarr(-9223372036854775808L). + doublearr(2344.0). + doublearr(123.0). + stringarr("bar"). + enumarr(Enumarr.VALUES). + refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + fileArr("bin"). + + basicStruct(new BasicStruct.Builder(). + //foo("basicFoo"). + bar(3). + intArr(310)). + + rootStruct(new RootStruct.Builder(). + inner0(new RootStruct.Inner0.Builder(). + index(11)). + inner1(new RootStruct.Inner1.Builder(). + index(12)). + innerArr(new RootStruct.InnerArr.Builder(). + //boolVal(true). + stringVal("deep"))). + + myarray(new Myarray.Builder(). + //intval(-5). + stringval("baah"). + stringval("yikes"). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file0"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(7)). + myStruct(new Myarray.MyStruct.Builder(). + a(1). + b(2))). + + myarray(new Myarray.Builder(). + intval(5). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file1"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(1). + foo(2)). + myStruct(new Myarray.MyStruct.Builder(). + a(-1). + b(-2))); + + } + + private FunctionTestConfig.Builder createVariableAccessBuilderManyUnset() { + return new FunctionTestConfig.Builder(). + bool_val(false). + bool_with_def(true). + //int_val(5). + int_with_def(-14). + long_val(12345678901L). + long_with_def(-9876543210L). + double_val(41.23). + double_with_def(-12). + string_val("foo"). + //stringwithdef("bar"). + enum_val(Enum_val.FOOBAR). + //enumwithdef(Enumwithdef.BAR2). + refval(":parent:"). + refwithdef(":parent:"). + fileVal("etc"). + boolarr(false). + longarr(9223372036854775807L). + longarr(-9223372036854775808L). + doublearr(2344.0). + doublearr(123.0). + stringarr("bar"). + enumarr(Enumarr.VALUES). + refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + fileArr("bin"). + + basicStruct(new BasicStruct.Builder(). + //foo("basicFoo"). + bar(3). + intArr(310)). + + rootStruct(new RootStruct.Builder(). + inner0(new RootStruct.Inner0.Builder(). + index(11)). + inner1(new RootStruct.Inner1.Builder(). + index(12)). + innerArr(new RootStruct.InnerArr.Builder(). + //boolVal(true). + stringVal("deep"))). + + myarray(new Myarray.Builder(). + intval(-5). + stringval("baah"). + stringval("yikes"). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file0"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(7)). + myStruct(new Myarray.MyStruct.Builder(). + a(1). + b(2))). + + myarray(new Myarray.Builder(). + intval(5). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file1"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(1). + foo(2)). + myStruct(new Myarray.MyStruct.Builder(). + a(-1). + b(-2))); + + } + + private InnerCNode getDef(String[] schema) { + ArrayList<String> def = new ArrayList<>(); + def.addAll(Arrays.asList(schema)); + return new DefParser("documentmanager", + new StringReader(StringUtilities.implode(def.toArray(new String[def.size()]), "\n"))).getTree(); + } + + @Test + public void testExtraFieldsAreIgnored() throws Exception { + try { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + InnerCNode defWithExtra = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n") + "\nnewfield string default=\"foo\"\n")).getTree(); + InstanceResolver.applyDef(builder, defWithExtra); + } catch (NoSuchFieldException e) { + fail("Should not fail on extra field"); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/RecentLogFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/RecentLogFilterTest.java new file mode 100644 index 00000000000..c203efb3d94 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/RecentLogFilterTest.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; + + +/** + * @author musum + * @since 5.1 + */ +public class RecentLogFilterTest { + + @Test + public void basic() { + RecentLogFilter rlf = new RecentLogFilter(); + List<LogRecord> logRecords = new ArrayList<>(); + for (int i = 0; i < RecentLogFilter.maxMessages + 1; i++) { + logRecords.add(new LogRecord(Level.INFO, "" + i)); + } + + assertTrue(rlf.isLoggable(logRecords.get(0))); + assertFalse(rlf.isLoggable(logRecords.get(0))); + + for (int i = 1; i < RecentLogFilter.maxMessages + 1; i++) { + assertTrue(rlf.isLoggable(logRecords.get(i))); + } + System.out.println(logRecords.size()); + System.out.println(logRecords); + + // Should have filled up maxMessages slots with records 1-maxMessages + // and pushed the first one out, so the below should return true + assertTrue(rlf.isLoggable(logRecords.get(0))); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java new file mode 100644 index 00000000000..05445642803 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -0,0 +1,257 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.MockModelContext; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.*; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.*; +import org.junit.Before; +import org.junit.Test; +import org.mockito.internal.stubbing.answers.ThrowsExceptionClass; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author lulf + */ +public class VespaModelFactoryTest { + + private ModelContext testModelContext; + + @Before + public void setupContext() { + testModelContext = new MockModelContext(); + } + + @Test + public void testThatFactoryCanBuildModel() { + VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + Model model = modelFactory.createModel(testModelContext); + assertNotNull(model); + assertTrue(model instanceof VespaModel); + } + + // Uses an application package that throws IllegalArgumentException when validating + @Test(expected = IllegalArgumentException.class) + public void testThatFactoryModelValidationFailsWithIllegalArgumentException() { + VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + modelFactory.createAndValidateModel(new MockModelContext(createApplicationPackageThatFailsWhenValidating()), false); + } + + // Uses a MockApplicationPackage that throws throws UnsupportedOperationException (rethrown as RuntimeException) when validating + @Test(expected = RuntimeException.class) + public void testThatFactoryModelValidationFails() { + VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + modelFactory.createAndValidateModel(testModelContext, false); + } + + @Test + public void testThatFactoryModelValidationCanBeIgnored() { + VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + ModelCreateResult createResult = modelFactory.createAndValidateModel( + new MockModelContext(createApplicationPackageThatFailsWhenValidating()), + true); + assertNotNull(createResult.getModel()); + assertNotNull(createResult.getConfigChangeActions()); + assertTrue(createResult.getConfigChangeActions().isEmpty()); + } + + @Test + public void hostedVespaRoutingApplicationAllocatesNodesWithHostsXml() { + String hostName = "test-host-name"; + String routingClusterName = "routing-cluster"; + + String hosts = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<hosts>\n" + + " <host name='" + hostName + "'>\n" + + " <alias>proxy1</alias>\n" + + " </host>\n" + + "</hosts>"; + + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0' xmlns:deploy='vespa'>\n" + + " <admin version='2.0'>\n" + + " <adminserver hostalias='proxy1' />\n" + + " </admin>" + + " <jdisc id='" + routingClusterName + "' version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='proxy1' />\n" + + " </nodes>\n" + + " </jdisc>\n" + + "</services>"; + + HostProvisioner provisionerToOverride = + mock(HostProvisioner.class, new ThrowsExceptionClass(UnsupportedOperationException.class)); + + ModelContext modelContext = new MockModelContext() { + @Override + public ApplicationPackage applicationPackage() { + return new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build(); + } + + @Override + public Optional<HostProvisioner> hostProvisioner() { + return Optional.of(provisionerToOverride); + } + + @Override + public Properties properties() { + return new Properties() { + @Override + public boolean multitenant() { + return true; + } + + @Override + public boolean hostedVespa() { + return true; + } + + @Override + public Zone zone() { + return Zone.defaultZone(); + } + + @Override + public Set<Rotation> rotations() { + return new HashSet<>(); + } + + @Override + public ApplicationId applicationId() { + return ApplicationId.HOSTED_ZONE_APPLICATION_ID; + } + + @Override + public List<ConfigServerSpec> configServerSpecs() { + return Collections.emptyList(); + } + }; + } + }; + + Model model = new VespaModelFactory(new NullConfigModelRegistry()).createModel(modelContext); + + List<HostInfo> allocatedHosts = new ArrayList<>(model.getHosts()); + assertThat(allocatedHosts.size(), is(1)); + HostInfo hostInfo = allocatedHosts.get(0); + + assertThat(hostInfo.getHostname(), is(hostName)); + + assertTrue("Routing service should run on host " + hostName, + hostInfo.getServices().stream() + .map(ServiceInfo::getConfigId) + .anyMatch(configId -> configId.contains(routingClusterName))); + } + + @Test + public void hostedVespaZoneApplicationAllocatesNodesWithHostsXml() { + String hostName = "test-host-name"; + String routingClusterName = "routing-cluster"; + + String hosts = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<hosts>\n" + + " <host name='" + hostName + "'>\n" + + " <alias>proxy1</alias>\n" + + " </host>\n" + + "</hosts>"; + + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0' xmlns:deploy='vespa'>\n" + + " <admin version='2.0'>\n" + + " <adminserver hostalias='proxy1' />\n" + + " </admin>" + + " <jdisc id='" + routingClusterName + "' version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='proxy1' />\n" + + " </nodes>\n" + + " </jdisc>\n" + + "</services>"; + + HostProvisioner provisionerToOverride = + mock(HostProvisioner.class, new ThrowsExceptionClass(UnsupportedOperationException.class)); + + ModelContext modelContext = new MockModelContext() { + @Override + public ApplicationPackage applicationPackage() { + return new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build(); + } + + @Override + public Optional<HostProvisioner> hostProvisioner() { + return Optional.of(provisionerToOverride); + } + + @Override + public Properties properties() { + return new Properties() { + @Override + public boolean multitenant() { + return true; + } + + @Override + public boolean hostedVespa() { + return true; + } + + @Override + public Zone zone() { + return Zone.defaultZone(); + } + + @Override + public Set<Rotation> rotations() { + return new HashSet<>(); + } + + @Override + public ApplicationId applicationId() { + return ApplicationId.HOSTED_ZONE_APPLICATION_ID; + } + + @Override + public List<ConfigServerSpec> configServerSpecs() { + return Collections.emptyList(); + } + }; + } + }; + + Model model = new VespaModelFactory(new NullConfigModelRegistry()).createModel(modelContext); + + List<HostInfo> allocatedHosts = new ArrayList<>(model.getHosts()); + assertThat(allocatedHosts.size(), is(1)); + HostInfo hostInfo = allocatedHosts.get(0); + + assertThat(hostInfo.getHostname(), is(hostName)); + + assertTrue("Routing service should run on host " + hostName, + hostInfo.getServices().stream() + .map(ServiceInfo::getConfigId) + .anyMatch(configId -> configId.contains(routingClusterName))); + } + + ApplicationPackage createApplicationPackageThatFailsWhenValidating() { + return new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().failOnValidateXml().build(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java new file mode 100644 index 00000000000..a8554c54867 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.SlobroksConfig.Slobrok; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.StatisticsComponent; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + + +/** + * @author gjoranv + */ +public class AdminTestCase { + + protected static final String TESTDIR = "src/test/cfg/admin/"; + + protected VespaModel getVespaModel(String configPath) { + return new VespaModelCreatorWithFilePkg(configPath).create(); + } + + /** + * Test that version 2.0 of adminconfig works as expected. + */ + @Test + public void testAdmin20() throws Exception { + VespaModel vespaModel = getVespaModel(TESTDIR + "adminconfig20"); + + // Verify that the admin plugin has been loaded (always loads routing). + assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + + ApplicationConfigProducerRoot root = vespaModel.getVespa(); + assertNotNull(root); + + // Verify configIds + Set<String> configIds = vespaModel.getConfigIds(); + String localhost = HostName.getLocalhost(); + String localhostConfigId = "hosts/" + localhost; + assertTrue(configIds.contains(localhostConfigId)); + assertTrue(configIds.contains("admin/logserver")); + assertTrue(configIds.contains("admin/configservers/configserver.0")); + assertTrue(configIds.contains("admin/slobrok.0")); + assertTrue(configIds.contains("admin/slobrok.1")); + assertFalse(configIds.contains("admin/slobrok.2")); + assertTrue(configIds.contains("admin")); + + // Confirm 2 slobroks in config + SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); + vespaModel.getConfig(sb, "admin/slobrok.0"); + SlobroksConfig sc = new SlobroksConfig(sb); + assertEquals(sc.slobrok().size(), 2); + boolean localHostOK = false; + for (Slobrok s : sc.slobrok()) { + if (s.connectionspec().matches(".*" + localhost + ".*")) localHostOK = true; + } + assertTrue(localHostOK); + + LogdConfig.Builder lb = new LogdConfig.Builder(); + vespaModel.getConfig(lb, "admin/slobrok.0"); + LogdConfig lc = new LogdConfig(lb); + assertEquals(lc.logserver().host(), localhost); + + // Verify services in the sentinel config + SentinelConfig.Builder b = new SentinelConfig.Builder(); + vespaModel.getConfig(b, localhostConfigId); + SentinelConfig sentinelConfig = new SentinelConfig(b); + assertThat(sentinelConfig.service().size(), is(5)); + assertThat(sentinelConfig.service(0).name(), is("logserver")); + assertThat(sentinelConfig.service(1).name(), is("slobrok")); + assertThat(sentinelConfig.service(2).name(), is("slobrok2")); + assertThat(sentinelConfig.service(3).name(), is("logd")); + assertThat(sentinelConfig.service(4).name(), is("filedistributorservice")); + } + + /** + * Test that a very simple config with only adminserver tag creates + * adminserver, logserver, configserver and slobroks + */ + @Test + public void testOnlyAdminserver() throws Exception { + VespaModel vespaModel = getVespaModel(TESTDIR + "simpleadminconfig20"); + + // Verify that the admin plugin has been loaded (always loads routing). + assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + + ApplicationConfigProducerRoot root = vespaModel.getVespa(); + assertNotNull(root); + + // Verify configIds + Set<String> configIds = vespaModel.getConfigIds(); + String localhost = HostName.getLocalhost(); + String localhostConfigId = "hosts/" + localhost; + assertTrue(configIds.contains(localhostConfigId)); + assertTrue(configIds.contains("admin/logserver")); + assertTrue(configIds.contains("admin/configservers/configserver.0")); + assertTrue(configIds.contains("admin/slobrok.0")); + assertFalse(configIds.contains("admin/slobrok.1")); + + // Verify services in the sentinel config + SentinelConfig.Builder b = new SentinelConfig.Builder(); + vespaModel.getConfig(b, localhostConfigId); + SentinelConfig sentinelConfig = new SentinelConfig(b); + assertThat(sentinelConfig.service().size(), is(4)); + assertThat(sentinelConfig.service(0).name(), is("logserver")); + assertThat(sentinelConfig.service(1).name(), is("slobrok")); + assertThat(sentinelConfig.service(2).name(), is("logd")); + assertThat(sentinelConfig.service(3).name(), is("filedistributorservice")); + assertThat(sentinelConfig.service(0).affinity().cpuSocket(), is(-1)); + assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); + + // Confirm slobrok config + SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); + vespaModel.getConfig(sb, "admin"); + SlobroksConfig sc = new SlobroksConfig(sb); + assertEquals(sc.slobrok().size(), 1); + assertTrue(sc.slobrok().get(0).connectionspec().matches(".*" + localhost + ".*")); + } + + @Test + public void testTenantAndAppInSentinelConfig() { + DeployState state = new DeployState.Builder().properties( + new DeployProperties.Builder(). + zone(new Zone(Environment.dev, RegionName.from("baz"))). + applicationId(new ApplicationId.Builder(). + tenant("quux"). + applicationName("foo").instanceName("bim").build()).build()).build(); + TestRoot root = new TestDriver().buildModel(state); + String localhost = HostName.getLocalhost(); + SentinelConfig config = root.getConfig(SentinelConfig.class, "hosts/" + localhost); + assertThat(config.application().tenant(), is("quux")); + assertThat(config.application().name(), is("foo")); + assertThat(config.application().environment(), is("dev")); + assertThat(config.application().region(), is("baz")); + assertThat(config.application().instance(), is("bim")); + } + + @Test + public void testMultipleConfigServers() throws Exception { + VespaModel vespaModel = getVespaModel(TESTDIR + "multipleconfigservers"); + + // Verify that the admin plugin has been loaded (always loads routing). + assertThat(vespaModel.configModelRepo().asMap().size(), is(2)); + ApplicationConfigProducerRoot root = vespaModel.getVespa(); + assertNotNull(root); + + Admin admin = vespaModel.getAdmin(); + assertNotNull(admin); + + // Verify configIds + Set<String> configIds = vespaModel.getConfigIds(); + String localhost = HostName.getLocalhost(); + String localhostConfigId = "hosts/" + localhost; + assertTrue(configIds.contains(localhostConfigId)); + assertTrue(configIds.contains("admin/logserver")); + assertTrue(configIds.contains("admin/configservers/configserver.0")); + assertTrue(configIds.contains("admin/configservers/configserver.1")); + + assertThat(admin.getConfigservers().size(), is(2)); + + // Default configserver is the first one in the list and should have the default ports too + Configserver server1 = admin.getConfigservers().get(0); + assertEquals(admin.getConfigserver(), server1); + assertThat(server1.getPortCount(), is(2)); + assertThat(server1.getRelativePort(0), is(19070)); + assertThat(server1.getRelativePort(1), is(19071)); + + + // Second configserver should be on second host but have the same port number + Configserver server2 = admin.getConfigservers().get(1); + + assertNotSame(server1, server2); + assertNotSame(server1.getHostName(), server2.getHostName()); + + assertThat(server2.getPortCount(), is(2)); + assertThat(server2.getRelativePort(0), is(19070)); + assertThat(server2.getRelativePort(1), is(19071)); + } + + @Test + public void testContainerMetricsSnapshotInterval() throws Exception { + VespaModel vespaModel = getVespaModel(TESTDIR + "metricconfig"); + + ContainerCluster docprocCluster = vespaModel.getContainerClusters().get("cluster.music.indexing"); + HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); + docprocCluster.getConfig(builder); + HealthMonitorConfig docprocConfig = new HealthMonitorConfig(builder); + assertEquals(60, (int) docprocConfig.snapshot_interval()); + + ContainerCluster qrCluster = vespaModel.getContainerClusters().get("container"); + builder = new HealthMonitorConfig.Builder(); + qrCluster.getConfig(builder); + HealthMonitorConfig qrClusterConfig = new HealthMonitorConfig(builder); + assertEquals(60, (int) qrClusterConfig.snapshot_interval()); + + StatisticsComponent stat = null; + for (Component component : qrCluster.getAllComponents()) { + System.out.println(component.getClassId().getName()); + if (component.getClassId().getName().contains("com.yahoo.statistics.StatisticsImpl")) { + stat = (StatisticsComponent) component; + break; + } + } + assertNotNull(stat); + StatisticsConfig.Builder sb = new StatisticsConfig.Builder(); + stat.getConfig(sb); + StatisticsConfig sc = new StatisticsConfig(sb); + assertEquals(60, (int) sc.collectionintervalsec()); + assertEquals(60, (int) sc.loggingintervalsec()); + } + + @Test + public void testStatisticsConfig() { + StatisticsComponent stat = new StatisticsComponent(); + StatisticsConfig.Builder sb = new StatisticsConfig.Builder(); + stat.getConfig(sb); + StatisticsConfig sc = new StatisticsConfig(sb); + assertEquals(sc.collectionintervalsec(), 300, 0.1); + assertEquals(sc.loggingintervalsec(), 300, 0.1); + assertEquals(sc.values(0).operations(0).name(), StatisticsConfig.Values.Operations.Name.REGULAR); + assertEquals(sc.values(0).operations(0).arguments(0).key(), "limits"); + assertEquals(sc.values(0).operations(0).arguments(0).value(), "25,50,100,500"); + + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java new file mode 100644 index 00000000000..200e51ef924 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -0,0 +1,440 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.config.model.application.provider.SimpleApplicationValidator; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Collection; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * Test for creating cluster controllers under the admin tag. + */ +public class ClusterControllerTestCase extends DomBuilderTest { + + private List<String> sds; + + @Before + public void setup() { + sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + } + + @Test + public void testSingleCluster() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <cluster-controllers>\n" + + " <cluster-controller hostalias=\"configserver\">" + + " </cluster-controller>" + + " <cluster-controller hostalias=\"configserver\"/>" + + " <cluster-controller hostalias=\"configserver\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " <tuning>" + + " <cluster-controller>\n" + + " <init-progress-time>34567s</init-progress-time>" + + " <transition-time>4000ms</transition-time>" + + " <stable-state-period>1h</stable-state-period>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + + VespaModel model = createVespaModel(xml); + assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); + + assertTrue(existsHostsWithClusterControllerConfigId(model)); + assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); + for (int i = 0; i < 3; ++i) { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + model.getConfig(builder, "admin/cluster-controllers/" + i + "/components/clustercontroller-bar-configurer"); + + FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); + assertThat(cfg.index(), is(i)); + assertThat(cfg.fleet_controller_count(), is(3)); + assertThat(cfg.init_progress_time(), is(34567000)); + assertThat(cfg.storage_transition_time(), is(4000)); + assertThat(cfg.stable_state_time_period(), is(3600000)); + } + } + + + @Test(expected = IllegalArgumentException.class) + public void testSeparateHostsRequired() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"mockhost\" />\n" + + " <cluster-controllers standalone-zookeeper=\"true\">\n" + + " <cluster-controller hostalias=\"mockhost\"/>" + + " <cluster-controller hostalias=\"mockhost\"/>" + + " <cluster-controller hostalias=\"mockhost\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " </documents>\n" + + " <group>" + + " <node hostalias='mockhost' distribution-key='0' />" + + " </group>" + + " </content>" + + "\n" + + "</services>"; + TestDriver driver = new TestDriver(); + driver.buildModel(xml); + } + + @Test(expected = IllegalArgumentException.class) + public void testSeparateHostsFromConfigServerRequired() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"mockhost\" />\n" + + " <configservers>\n" + + " <configserver hostalias=\"mockhost\" />" + + " </configservers>" + + " <cluster-controllers standalone-zookeeper=\"true\">\n" + + " <cluster-controller hostalias=\"mockhost\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " </documents>\n" + + " <group>" + + " <node hostalias='mockhost' distribution-key='0' />" + + " </group>" + + " </content>" + + "\n" + + "</services>"; + TestDriver driver = new TestDriver(); + driver.buildModel(xml); + } + + @Test + public void testStandaloneZooKeeper() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"node1\" />\n" + + " <cluster-controllers standalone-zookeeper=\"true\">\n" + + " <cluster-controller hostalias=\"node2\"/>" + + " <cluster-controller hostalias=\"node3\"/>" + + " <cluster-controller hostalias=\"node4\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " </documents>\n" + + " <group>" + + " <node hostalias='node1' distribution-key='0' />" + + " </group>" + + " </content>" + + "\n" + + "</services>"; + String hosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<hosts>\n" + + " <host name=\"localhost\">\n" + + " <alias>node1</alias>\n" + + " </host>\n" + + " <host name=\"my.host1.com\">\n" + + " <alias>node2</alias>\n" + + " </host>\n" + + " <host name=\"my.host2.com\">\n" + + " <alias>node3</alias>\n" + + " </host>\n" + + " <host name=\"my.host3.com\">\n" + + " <alias>node4</alias>\n" + + " </host>\n" + + "</hosts>"; + TestDriver driver = new TestDriver(); + TestRoot root = driver.buildModel(xml, hosts); + assertZookeepersConfig(root); + assertZookeeperServerConfig(root, 0); + assertZookeeperServerConfig(root, 1); + assertZookeeperServerConfig(root, 2); + } + + private void assertZookeepersConfig(TestRoot root) { + ZookeepersConfig.Builder builder = new ZookeepersConfig.Builder(); + root.getConfig(builder, "admin/standalone"); + ZookeepersConfig config = new ZookeepersConfig(builder); + assertThat(config.zookeeperserverlist().split(",").length, is(3)); + } + + private void assertZookeeperServerConfig(TestRoot root, int id) { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + root.getConfig(builder, "admin/standalone/cluster-controllers/" + id); + ZookeeperServerConfig config = new ZookeeperServerConfig(builder); + assertThat(config.server().size(), is(3)); + assertThat(config.myid(), is(id)); + Collection<Integer> serverIds = Collections2.transform(config.server(), new Function<ZookeeperServerConfig.Server, Integer>() { + @Override + public Integer apply(ZookeeperServerConfig.Server server) { + return server.id(); + } + }); + assertTrue(serverIds.contains(id)); + } + + + @Test + public void testUnconfigured() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <logserver hostalias=\"logserver\" />\n" + + " <slobroks>\n" + + " <slobrok hostalias=\"configserver\" />\n" + + " <slobrok hostalias=\"logserver\" />\n" + + " </slobroks>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " <tuning>" + + " <cluster-controller>\n" + + " <init-progress-time>34567</init-progress-time>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + VespaModel model = createVespaModel(xml); + assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); + + assertTrue(existsHostsWithClusterControllerConfigId(model)); + assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); + assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(1)); + + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + model.getConfig(builder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); + + FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); + assertThat(cfg.index(), is(0)); + assertThat(cfg.fleet_controller_count(), is(1)); + assertThat(cfg.init_progress_time(), is(34567000)); + } + + private boolean existsHostsWithClusterControllerConfigId(VespaModel model) { + boolean found = false; + for (HostResource h : model.getHostSystem().getHosts()) { + for (Service s : h.getServices()) { + if (s.getConfigId().equals("admin/cluster-controllers/0")) { + found = true; + } + } + } + return found; + } + + @Test + public void testUnconfiguredMultiple() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <configservers>\n" + + " <configserver hostalias=\"node0\"/>" + + " <configserver hostalias=\"node1\"/>" + + " <configserver hostalias=\"node2\"/>" + + " </configservers>\n" + + " <slobroks>\n" + + " <slobrok hostalias=\"configserver\" />\n" + + " <slobrok hostalias=\"logserver\" />\n" + + " </slobroks>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " <tuning>\n" + + " <cluster-controller>" + + " <init-progress-time>34567</init-progress-time>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + VespaModel model = createVespaModel(xml); + + assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(3)); + assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); + assertGroupSize(model, "admin/cluster-controllers/1/components/clustercontroller-bar-configurer", 1); + assertGroupSize(model, "admin/cluster-controllers/2/components/clustercontroller-bar-configurer", 1); + } + + @Test + public void testUnconfiguredNoTuning() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <logserver hostalias=\"logserver\" />\n" + + " <slobroks>\n" + + " <slobrok hostalias=\"configserver\" />\n" + + " <slobrok hostalias=\"logserver\" />\n" + + " </slobroks>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " </content>" + + "\n" + + "</services>"; + + VespaModel model = createVespaModel(xml); + assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); + + assertTrue(existsHostsWithClusterControllerConfigId(model)); + assertGroupSize(model, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer", 1); + assertThat(model.getAdmin().getClusterControllers().getContainers().size(), is(1)); + + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + model.getConfig(builder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); + + FleetcontrollerConfig cfg = new FleetcontrollerConfig(builder); + assertThat(cfg.index(), is(0)); + } + + @Test + public void testUnconfiguredNoContent() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " </admin>\n" + + " <container version=\"1.0\">\n" + + " <nodes>" + + " <node hostalias=\"node1\"/>\n" + + " </nodes>\n" + + " </container>\n" + + "</services>"; + + VespaModel model = createVespaModel(xml); + assertFalse(model.getService("admin/cluster-controllers/0").isPresent()); + + assertFalse(existsHostsWithClusterControllerConfigId(model)); + assertNull(model.getAdmin().getClusterControllers()); + } + + @Test + public void testUsingOldStyle() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <logserver hostalias=\"logserver\" />\n" + + " <slobroks>\n" + + " <slobrok hostalias=\"configserver\" />\n" + + " <slobrok hostalias=\"logserver\" />\n" + + " </slobroks>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " <tuning>\n" + + " <cluster-controller>" + + " <init-progress-time>34567</init-progress-time>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + VespaModel model = createVespaModel(xml); + assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); + + assertTrue(existsHostsWithClusterControllerConfigId(model)); + { + //StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + //try { + // model.getConfig(builder, "admin/cluster-controllers/0/components/bar-configurer"); + // fail("Invalid config id didn't fail."); + //} catch (UnknownConfigIdException e) { + // assertTrue(e.getMessage().matches(".*Invalid config id.*")); + //} + } + } + + private void assertGroupSize(VespaModel model, String configId, int size) { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + model.getConfig(builder, configId); + StorDistributionConfig cfg = new StorDistributionConfig(builder); + assertThat(cfg.group().size(), is(size)); + } + + private VespaModel createVespaModel(String servicesXml) throws IOException, SAXException { + VespaModel model = new VespaModel(new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withSearchDefinitions(sds) + .build()); + SimpleApplicationValidator.checkServices(new StringReader(servicesXml)); + return model; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java new file mode 100644 index 00000000000..b4e366c1609 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.Hosts; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @author bratseth + */ +public class DedicatedAdminV4Test { + + private static final String services = + "<services>" + + " <admin version='4.0'>" + + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + + " <logservers><nodes count='1' dedicated='true'/></logservers>" + + " </admin>" + + "</services>"; + + @Test + public void testModelBuilding() throws IOException, SAXException { + String hosts = "<hosts>" + + " <host name=\"myhost0\">" + + " <alias>node0</alias>" + + " </host>" + + " <host name=\"myhost1\">" + + " <alias>node1</alias>" + + " </host>" + + " <host name=\"myhost2\">" + + " <alias>node2</alias>" + + " </host>" + + "</hosts>"; + ApplicationPackage app = new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder().applicationPackage(app).modelHostProvisioner(new InMemoryProvisioner(Hosts.getHosts(app.getHosts()), true)).build()); + assertEquals(3, model.getHosts().size()); + + Set<String> serviceNames0 = serviceNames(model.getConfig(SentinelConfig.class, "hosts/myhost0")); + assertEquals(3, serviceNames0.size()); + assertTrue(serviceNames0.contains("slobrok")); + assertTrue(serviceNames0.contains("logd")); + assertTrue(serviceNames0.contains("filedistributorservice")); + + Set<String> serviceNames1 = serviceNames(model.getConfig(SentinelConfig.class, "hosts/myhost1")); + assertEquals(3, serviceNames1.size()); + assertTrue(serviceNames1.contains("slobrok")); + assertTrue(serviceNames1.contains("logd")); + assertTrue(serviceNames1.contains("filedistributorservice")); + + Set<String> serviceNames2 = serviceNames(model.getConfig(SentinelConfig.class, "hosts/myhost2")); + assertEquals(3, serviceNames2.size()); + assertTrue(serviceNames2.contains("logserver")); + assertTrue(serviceNames2.contains("logd")); + assertTrue(serviceNames2.contains("filedistributorservice")); + } + + private Set<String> serviceNames(SentinelConfig config) { + return config.service().stream().map(SentinelConfig.Service::name).collect(Collectors.toSet()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java new file mode 100644 index 00000000000..3efd7e171d3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import org.junit.Test; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.BaseDeployLogger; + +import java.io.File; +import java.io.IOException; +import java.util.jar.JarFile; +import java.util.logging.Level; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class ComponentValidatorTest { + private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; + + @Test + public void basicComponentValidation() throws Exception { + // Valid jar file + JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); + ComponentValidator componentValidator = new ComponentValidator(ok); + componentValidator.validateAll(new BaseDeployLogger()); + + // No manifest + validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); + } + + private void validateWithException(String jarName, String exceptionMessage) throws IOException { + try { + JarFile jarFile = new JarFile(JARS_DIR + jarName); + ComponentValidator componentValidator = new ComponentValidator(jarFile); + componentValidator.validateAll(new BaseDeployLogger()); + assert (false); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is(exceptionMessage)); + } + } + + @Test + public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { + final StringBuffer buffer = new StringBuffer(); + + DeployLogger logger = new DeployLogger() { + @Override + public void log(Level level, String message) { + buffer.append(message).append('\n'); + } + }; + + new ComponentValidator(new JarFile(JARS_DIR + "snapshot_bundle.jar")).validateAll(logger); + assertThat(buffer.toString(), containsString("Deploying snapshot bundle")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java new file mode 100644 index 00000000000..bfd25b8587c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Harald Musum</a> + */ +public class DeploymentFileValidatorTest { + + @Test + public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException { + final String simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts> " + + "<host name=\"localhost\">" + + "<alias>node0</alias>" + + "</host>" + + "</hosts>"; + + final String services = "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node0'/>" + + " </nodes>" + + " </jdisc>" + + "</services>"; + + final String deploymentInfo = "<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod global-service-id='non-existing'>" + + " <region active='true'>us-east</region>" + + " </prod>" + + "</deployment>"; + + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts(simpleHosts) + .withServices(services) + .withDeploymentInfo(deploymentInfo) + .build(); + DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); + try { + final DeployState deployState = builder.build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new DeploymentFileValidator().validate(model, deployState); + fail("Did not get expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexesTest.java new file mode 100644 index 00000000000..a9532fb52b3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexesTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Harald Musum</a> + */ +public class NoPrefixForIndexesTest { + + @Test + public void requireThatPrefixIsSupported() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/prefix/").create(); + } + + @Test + public void requireThatPrefixIsSupportedForStreaming() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/prefix_streaming/").create(); + } + + @Test + public void requireThatPrefixIsIllegalForIndexField() { + try { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/prefix_index/").create(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'simple', field 'artist': match/index:prefix is not supported for indexes.", e.getMessage()); + } + } + + @Test + public void requireThatPrefixIsIllegalForMixedAttributeAndIndexField() { + try { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/prefix_index_and_attribute/").create(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For search 'simple', field 'artist': match/index:prefix is not supported for indexes.", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java new file mode 100644 index 00000000000..8c1a288c46d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidatorTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchDataTypeValidatorTestCase { + + @Test + public void requireThatSupportedTypesAreValidated() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/search_alltypes/").create(); + } + + @Test + public void requireThatStructsAreLegalInSearchClusters() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/search_struct/").create(); + } + + @Test + public void requireThatEmptyContentFieldIsLegalInSearchClusters() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/search_empty_content/").create(); + } + + @Test + public void requireThatIndexingMapsInNonStreamingClusterIsIllegal() { + try { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/index_struct/").create(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Field type 'Map<string,string>' cannot be indexed for search clusters (field 'baz' in definition " + + "'simple' for cluster 'content').", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverrideTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverrideTest.java new file mode 100644 index 00000000000..3354e9320fc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationOverrideTest.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Optional; + +import static org.junit.Assert.fail; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ValidationOverrideTest { + + @Test + public void testValidationOverridesInIsolation() throws IOException, SAXException { + String validationOverrides = + "<validation-overrides>" + + " <allow until='2000-01-01'>indexing-change</allow>" + + " <allow until='2000-01-03' comment='any text'>indexing-mode-change</allow>" + + "</validation-overrides>"; + + { + + ValidationOverrides overrides = new ValidationOverridesXMLReader().read(Optional.of(new StringReader(validationOverrides)), + at("2000-01-01T23:59:00")); + assertOverridden("indexing-change", overrides); + assertOverridden("indexing-mode-change", overrides); + assertNotOverridden("field-type-change", overrides); + } + + { + ValidationOverrides overrides = new ValidationOverridesXMLReader().read(Optional.of(new StringReader(validationOverrides)), + at("2000-01-02T00:00:00")); + assertNotOverridden("indexing-change", overrides); + assertOverridden("indexing-mode-change", overrides); + assertNotOverridden("field-type-change", overrides); + } + + { + ValidationOverrides overrides = new ValidationOverridesXMLReader().read(Optional.of(new StringReader(validationOverrides)), + at("2000-01-04T00:00:00")); + assertNotOverridden("indexing-change", overrides); + assertNotOverridden("indexing-mode-change", overrides); + assertNotOverridden("field-type-change", overrides); + } + + } + + @Test + public void testInvalidOverridePeriod() throws IOException, SAXException { + String validationOverrides = + "<validation-overrides>" + + " <allow until='2000-02-02'>indexing-change</allow>" + + "</validation-overrides>"; + + try { + new ValidationOverridesXMLReader().read(Optional.of(new StringReader(validationOverrides)), + at("2000-01-01T23:59:00")); + fail("Expected validation interval override validation validation failure"); + } + catch (IllegalArgumentException e) { + assertEquals("validation-overrides is invalid", e.getMessage()); + assertEquals("allow 'indexing-change' until 2000-02-03T00:00:00Z is too far in the future: Max 30 days is allowed", + e.getCause().getMessage()); + } + } + + private Instant at(String utcIsoTime) { + return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant(); + } + + private void assertOverridden(String validationId, ValidationOverrides overrides) { + overrides.invalid(ValidationId.from(validationId).get(), "message"); // should not throw exception + } + + private void assertNotOverridden(String validationId, ValidationOverrides overrides) { + try { + overrides.invalid(ValidationId.from(validationId).get(), "message"); + fail("Expected '" + validationId + "' to not be overridden"); + } + catch (ValidationOverrides.ValidationException expected) { + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java new file mode 100644 index 00000000000..75f4eb3e15d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * @author bratseth + */ +public class ValidationTester { + + private final int nodeCount; + + /** Creates a validation tester with 1 node available */ + public ValidationTester() { + this(1); + } + + /** Creates a validation tester with a number of nodes available */ + public ValidationTester(int nodeCount) { + this.nodeCount = nodeCount; + } + + /** + * Deploys an application + * + * @param previousModel the previous model, or null if no previous + * @param services the services file content + * @param validationOverrides the validation overrides file content, or null if none + * @return the new model and any change actions + */ + public Pair<VespaModel, List<ConfigChangeAction>> deploy(VespaModel previousModel, String services, String validationOverrides) { + Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); + ApplicationPackage newApp = new MockApplicationPackage.Builder() + .withServices(services) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withValidationOverrides(validationOverrides) + .build(); + VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); + DeployState.Builder deployStateBuilder = new DeployState.Builder() + .applicationPackage(newApp) + .properties(new DeployProperties.Builder().hostedVespa(true).build()) + .modelHostProvisioner(new InMemoryProvisioner(nodeCount)) + .now(now); + if (previousModel != null) + deployStateBuilder.previousModel(previousModel); + VespaModel newModel = newModelCreator.create(deployStateBuilder); + return new Pair<>(newModel, newModelCreator.configChangeActions); + } + + + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java new file mode 100644 index 00000000000..5126ada7818 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ConfigChangeRefeedAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class ClusterSizeReductionValidatorTest { + + @Test + public void testSizeReductionValidation() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(30); + + VespaModel previous = tester.deploy(null, getServices(30), null).getFirst(); + try { + tester.deploy(previous, getServices(14), null); + fail("Expected exception due to cluster size reduction"); + } + catch (IllegalArgumentException expected) { + assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testSizeReductionValidationMinimalDecreaseIsAllowed() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(30); + + VespaModel previous = tester.deploy(null, getServices(3), null).getFirst(); + tester.deploy(previous, getServices(2), null); + } + + /* + @Test + public void testSizeReductionTo50PercentIsAllowed() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(30); + + VespaModel previous = tester.deploy(null, getServices(30), null).getFirst(); + tester.deploy(previous, getServices(15), null); + } + */ + + @Test + public void testOverridingSizereductionValidation() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(30); + + VespaModel previous = tester.deploy(null, getServices(30), null).getFirst(); + tester.deploy(previous, getServices(14), sizeReductionOverride); // Allowed due to override + } + + private static String getServices(int size) { + return "<services version='1.0'>" + + " <content id='default' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='" + size + "'/>" + + " </content>" + + "</services>"; + } + + private static final String sizeReductionOverride = + "<validation-overrides>\n" + + " <allow until='2000-01-03'>cluster-size-reduction</allow>\n" + + "</validation-overrides>\n"; + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java new file mode 100644 index 00000000000..8f1b0ca1cc1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; + +import java.util.List; + +public class ConfigChangeTestUtils { + + public static VespaConfigChangeAction newRestartAction(String message) { + return new VespaRestartAction(message); + } + + public static VespaConfigChangeAction newRestartAction(String message, List<ServiceInfo> services) { + return new VespaRestartAction(message, services); + } + + public static VespaConfigChangeAction newRefeedAction(String name, ValidationOverrides overrides, String message) { + return VespaRefeedAction.of(name, overrides, message); + } + + public static VespaConfigChangeAction newRefeedAction(String name, ValidationOverrides overrides, String message, + List<ServiceInfo> services, String documentType) { + return VespaRefeedAction.of(name, overrides, message, services, documentType); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java new file mode 100644 index 00000000000..4375b5661b2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.test.AnotherrestartConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.test.RestartConfig; +import com.yahoo.test.SimpletypesConfig; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Testing the validator on both a stub model and a real-life Vespa model. + * + * @author bjorncs + */ +public class ConfigValueChangeValidatorTest { + + private DeployLoggerStub logger; + + @Before + public void resetLogger() { + logger = new DeployLoggerStub(); + } + + /** + * NOTE: This test method has the following assumptions about the {@link com.yahoo.vespa.model.VespaModel}: + * 1) verbosegc and heapsize in qr-start.def is marked with restart. + * 2) {@link com.yahoo.vespa.model.container.Container} class has QrStartConfig listed in its + * {@link com.yahoo.vespa.model.application.validation.RestartConfigs} attribute. + * 3) That the config ids for the container services have a specific value. + * + * This test will to a certain degree ensure that the annotations in the VespaModel is correctly applied. + */ + @Test + public void requireThatValidatorHandlesVespaModel() { + List<ConfigChangeAction> changes = getConfigChanges( + createVespaModel(createQrStartConfigSegment(true, 2096)), + createVespaModel(createQrStartConfigSegment(false, 2096)) + ); + assertEquals(3, changes.size()); + assertComponentsEquals(changes, "default/container.0", 0); + assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); + assertComponentsEquals(changes, "docproc/cluster.basicsearch.indexing/0", 2); + } + + @Test + public void requireThatValidatorDetectsConfigChangeFromService() { + MockRoot oldRoot = createRootWithChildren(new SimpleConfigProducer("p", 0) + .withChildren(new ServiceWithAnnotation("s1", 1), new ServiceWithAnnotation("s2", 2))); + MockRoot newRoot = createRootWithChildren(new SimpleConfigProducer("p", 0) + .withChildren(new ServiceWithAnnotation("s1", 3), new ServiceWithAnnotation("s2", 4))); + List<ConfigChangeAction> changes = getConfigChanges(oldRoot, newRoot); + assertEquals(2, changes.size()); + assertComponentsEquals(changes, "p/s1", 0); + assertComponentsEquals(changes, "p/s2", 1); + assertEquals("anotherrestart.anothervalue has changed from 1 to 3", changes.get(0).getMessage()); + assertEquals("anotherrestart.anothervalue has changed from 2 to 4", changes.get(1).getMessage()); + assertEmptyLog(); + } + + @Test + public void requireThatValidatorDetectsConfigChangeFromParentProducer() { + MockRoot oldRoot = createRootWithChildren(new SimpleConfigProducer("p", 1) + .withChildren(new ServiceWithAnnotation("s1", 0), new ServiceWithAnnotation("s2", 0))); + MockRoot newRoot = createRootWithChildren(new SimpleConfigProducer("p", 2) + .withChildren(new ServiceWithAnnotation("s1", 0), new ServiceWithAnnotation("s2", 0))); + List<ConfigChangeAction> changes = getConfigChanges(oldRoot, newRoot); + assertEquals(2, changes.size()); + assertComponentsEquals(changes, "p/s1", 0); + assertComponentsEquals(changes, "p/s2", 1); + assertEmptyLog(); + } + + @Test + public void requireThatValidatorHandlesModelsWithDifferentTopology() { + MockRoot oldRoot = createRootWithChildren( + new SimpleConfigProducer("p1", 0).withChildren(new ServiceWithAnnotation("s1", 1)), + new SimpleConfigProducer("p2", 0).withChildren(new ServiceWithAnnotation("s2", 1))); + MockRoot newRoot = createRootWithChildren( + new ServiceWithAnnotation("s1", 2), + new ServiceWithAnnotation("s2", 2), + new ServiceWithAnnotation("s3", 2) + ); + + List<ConfigChangeAction> changes = getConfigChanges(oldRoot, newRoot); + assertTrue(changes.isEmpty()); + assertEmptyLog(); + } + + @Test(expected = IllegalStateException.class) + public void requireThatAnnotationDoesNotHaveEmtpyConfigList() { + MockRoot root = createRootWithChildren(new EmptyConfigListAnnotationService("")); + getConfigChanges(root, root); + } + + @Test(expected = IllegalStateException.class) + public void requireThatConfigHasRestartMethods() { + MockRoot root = createRootWithChildren(new ConfigWithMissingMethodsAnnotatedService("")); + getConfigChanges(root, root); + } + + @Test + public void requireThatServicesAnnotatedWithNonRestartConfigProduceWarningInLog() { + MockRoot root = createRootWithChildren(new NonRestartConfigAnnotatedService("")); + getConfigChanges(root, root); + assertEquals(1, logger.entries.size()); + } + + @Test + public void requireThatConfigsFromAnnotatedSuperClassesAreDetected() { + MockRoot oldRoot = createRootWithChildren(new SimpleConfigProducer("p", 1).withChildren( + new ChildServiceWithAnnotation("child1", 0), + new ChildServiceWithoutAnnotation("child2", 0))); + MockRoot newRoot = createRootWithChildren(new SimpleConfigProducer("p", 2).withChildren( + new ChildServiceWithAnnotation("child1", 0), + new ChildServiceWithoutAnnotation("child2", 0))); + List<ConfigChangeAction> changes = getConfigChanges(oldRoot, newRoot); + assertEquals(2, changes.size()); + assertComponentsEquals(changes, "p/child1", 0); + assertComponentsEquals(changes, "p/child2", 1); + assertEmptyLog(); + } + + private List<ConfigChangeAction> getConfigChanges(VespaModel currentModel, VespaModel nextModel) { + ConfigValueChangeValidator validator = new ConfigValueChangeValidator(logger); + return validator.validate(currentModel, nextModel, ValidationOverrides.empty()); + } + + private List<ConfigChangeAction> getConfigChanges(AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { + ConfigValueChangeValidator validator = new ConfigValueChangeValidator(logger); + return validator.findConfigChangesFromModels(currentModel, nextModel).collect(Collectors.toList()); + } + + private static void assertComponentsEquals(List<ConfigChangeAction> changes, String name, int index) { + assertEquals(name, changes.get(index).getServices().get(0).getConfigId()); + } + + private void assertEmptyLog() { + assertTrue(logger.entries.isEmpty()); + } + + private static VespaModel createVespaModel(String configSegment) { + // Note that the configSegment is here located on root. + return new VespaModelCreatorWithMockPkg( + null, + "<services version='1.0'>\n" + + configSegment + + " <admin version='2.0'>\n" + + " <adminserver hostalias='node1'/>\n" + + " </admin>\n" + + " <jdisc id='default' version='1.0'>\n" + + " <search/>\n" + + " <nodes>\n" + + " <node hostalias='node1'/>\n" + + " </nodes>\n" + + " </jdisc>\n" + + " <content id='basicsearch' version='1.0'>\n" + + " <redundancy>1</redundancy>\n" + + " <documents>\n" + + " <document type='music' mode='index'/>\n" + + " </documents>\n" + + " <group>\n" + + " <node hostalias='node1' distribution-key='0'/>\n" + + " </group>\n" + + " <engine>\n" + + " <proton>\n" + + " <searchable-copies>1</searchable-copies>\n" + + " </proton>\n" + + " </engine>\n" + + " </content>\n" + + "</services>", + Collections.singletonList("search music { document music { } }") + ).create(); + } + + private static String createQrStartConfigSegment(boolean verboseGc, int heapsize) { + return "<config name='search.config.qr-start'>\n" + + " <jvm>\n" + + " <verbosegc>" + verboseGc + "</verbosegc>\n" + + " <heapsize>" + heapsize + "</heapsize>\n" + + " </jvm>" + + "</config>\n"; + } + + private static MockRoot createRootWithChildren(AbstractConfigProducer<?>... children) { + MockRoot root = new MockRoot(); + Arrays.asList(children).forEach(root::addChild); + root.freezeModelTopology(); + return root; + } + + private static class NonRestartConfig extends ConfigInstance {} + + private static abstract class ServiceStub extends AbstractService { + public ServiceStub(String name) { + super(name); + setHostResource(new HostResource(new Host(null, "localhost"))); + } + + @Override + public int getPortCount() { + return 0; + } + } + + private static class SimpleConfigProducer extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements RestartConfig.Producer { + public final int value; + + public SimpleConfigProducer(String name, int value) { + super(name); + this.value = value; + } + + @Override + public void getConfig(RestartConfig.Builder builder) { + builder.value(value); + } + + public SimpleConfigProducer withChildren(AbstractConfigProducer<?>... producer) { + Arrays.asList(producer).forEach(this::addChild); + return this; + } + } + + + @RestartConfigs({RestartConfig.class, AnotherrestartConfig.class}) + private static class ServiceWithAnnotation extends ServiceStub implements AnotherrestartConfig.Producer { + public final int anotherValue; + + public ServiceWithAnnotation(String name, int anotherValue) { + super(name); + this.anotherValue = anotherValue; + } + + @Override + public void getConfig(AnotherrestartConfig.Builder builder) { + builder.anothervalue(anotherValue); + } + } + + @RestartConfigs(AnotherrestartConfig.class) + private static class ChildServiceWithAnnotation extends ServiceWithAnnotation { + public ChildServiceWithAnnotation(String name, int anotherValue) { + super(name, anotherValue); + } + } + + private static class ChildServiceWithoutAnnotation extends ServiceWithAnnotation { + public ChildServiceWithoutAnnotation(String name, int anotherValue) { + super(name, anotherValue); + } + } + + @RestartConfigs(SimpletypesConfig.class) + private static class NonRestartConfigAnnotatedService extends ServiceStub { + public NonRestartConfigAnnotatedService(String name) { + super(name); + } + } + + @RestartConfigs(NonRestartConfig.class) + private static class ConfigWithMissingMethodsAnnotatedService extends ServiceStub { + public ConfigWithMissingMethodsAnnotatedService(String name) { + super(name); + } + } + + @RestartConfigs + private static class EmptyConfigListAnnotationService extends ServiceStub { + public EmptyConfigListAnnotationService(String name) { + super(name); + } + } +} + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java new file mode 100644 index 00000000000..88ba6d885b8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidatorTest.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class ContainerRestartValidatorTest { + + @Test + public void validator_returns_action_for_containers_with_restart_on_deploy_enabled() { + VespaModel current = createModel(true); + VespaModel next = createModel(true); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(2, result.size()); + } + + @Test + public void validator_returns_empty_list_for_containers_with_restart_on_deploy_disabled() { + VespaModel current = createModel(false); + VespaModel next = createModel(false); + List<ConfigChangeAction> result = validateModel(current, next); + assertTrue(result.isEmpty()); + } + + private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { + return new ContainerRestartValidator() + .validate(current, next, new ValidationOverrides(Collections.emptyList())); + } + + private static VespaModel createModel(boolean restartOnDeploy) { + return new VespaModelCreatorWithMockPkg( + null, + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + " <jdisc id='cluster1' version='1.0'>\n" + + " <http>\n" + + " <server id='server1' port='" + Defaults.getDefaults().vespaWebServicePort() + "'/>\n" + + " </http>\n" + + " <config name='container.qr'>\n" + + " <restartOnDeploy>" + restartOnDeploy + "</restartOnDeploy>\n" + + " </config>\n" + + " </jdisc>\n" + + " <jdisc id='cluster2' version='1.0'>\n" + + " <http>\n" + + " <server id='server2' port='4090'/>\n" + + " </http>\n" + + " <config name='container.qr'>\n" + + " <restartOnDeploy>" + restartOnDeploy + "</restartOnDeploy>\n" + + " </config>\n" + + " </jdisc>\n" + + " <jdisc id='cluster3' version='1.0'>\n" + + " <http>\n" + + " <server id='server3' port='4100'/>\n" + + " </http>\n" + + " </jdisc>\n" + + "</services>" + ).create(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java new file mode 100644 index 00000000000..1b0fc00cbbc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class ContentClusterRemovalValidatorTest { + + @Test + public void testContentRemovalValidation() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(); + + VespaModel previous = tester.deploy(null, getServices("contentClusterId"), null).getFirst(); + try { + tester.deploy(previous, getServices("newContentClusterId"), null); + fail("Expected exception due to content cluster id change"); + } + catch (IllegalArgumentException expected) { + assertEquals("content-cluster-removal: Content cluster 'contentClusterId' is removed. This will cause loss of all data in this cluster", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOverridingContentRemovalValidation() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(); + + VespaModel previous = tester.deploy(null, getServices("contentClusterId"), null).getFirst(); + tester.deploy(previous, getServices("newContentClusterId"), removalOverride); // Allowed due to override + } + + private static String getServices(String contentClusterId) { + return "<services version='1.0'>" + + " <content id='" + contentClusterId + "' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='1'/>" + + " </content>" + + "</services>"; + } + + private static final String removalOverride = + "<validation-overrides>\n" + + " <allow until='2000-01-03'>content-cluster-removal</allow>\n" + + "</validation-overrides>\n"; + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java new file mode 100644 index 00000000000..8e15ca27a61 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; + +public class IndexedSearchClusterChangeValidatorTest { + + static class Fixture { + VespaModel currentModel; + VespaModel nextModel; + IndexedSearchClusterChangeValidator validator; + + public Fixture(VespaModel currentModel, VespaModel nextModel) { + this.currentModel = currentModel; + this.nextModel = nextModel; + validator = new IndexedSearchClusterChangeValidator(); + } + + public static Fixture newOneDocFixture(String currentSd, String nextSd) { + return new Fixture(newOneDocModel(currentSd), newOneDocModel(nextSd)); + } + + public static VespaModel newOneDocModel(String sdContent) { + return new ApplicationPackageBuilder(). + addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1"))). + addSearchDefinition(new SearchDefinitionBuilder(). + name("d1").content(sdContent).build()).buildCreator().create(); + } + + public static Fixture newTwoDocFixture(String currentSd, String nextSd) { + return new Fixture(newTwoDocModel(currentSd, currentSd), newTwoDocModel(nextSd, nextSd)); + } + + public static VespaModel newTwoDocModel(String d1Content, String d2Content) { + return new ApplicationPackageBuilder(). + addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1", "d2"))). + addSearchDefinition(new SearchDefinitionBuilder(). + name("d1").content(d1Content).build()). + addSearchDefinition(new SearchDefinitionBuilder(). + name("d2").content(d2Content).build()). + buildCreator().create(); + } + + public static Fixture newTwoClusterFixture(String currentSd, String nextSd) { + return new Fixture(newTwoClusterModel(currentSd, currentSd), newTwoClusterModel(nextSd, nextSd)); + } + + public static VespaModel newTwoClusterModel(String d1Content, String d2Content) { + return new ApplicationPackageBuilder(). + addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList("d1"))). + addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList("d2"))). + addSearchDefinition(new SearchDefinitionBuilder(). + name("d1").content(d1Content).build()). + addSearchDefinition(new SearchDefinitionBuilder(). + name("d2").content(d2Content).build()). + buildCreator().create(); + } + + public void assertValidation() { + List<ConfigChangeAction> act = normalizeServicesInActions(validator.validate(currentModel, nextModel, + ValidationOverrides.empty())); + assertThat(act.size(), is(0)); + } + + private static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { + return result.stream(). + map(action -> ((VespaConfigChangeAction) action).modifyAction( + action.getMessage(), + normalizeServices(action.getServices()), + action.getType().equals(ConfigChangeAction.Type.REFEED) ? + ((VespaRefeedAction)action).getDocumentType() : "")). + collect(Collectors.toList()); + } + + private static List<ServiceInfo> normalizeServices(List<ServiceInfo> services) { + return services.stream(). + map(service -> new ServiceInfo(service.getServiceName(), "null", null, null, + service.getConfigId(), "null")). + collect(Collectors.toList()); + } + + public void assertValidation(ConfigChangeAction exp) { + assertValidation(Arrays.asList(exp)); + } + + public void assertValidation(List<ConfigChangeAction> exp) { + List<ConfigChangeAction> act = normalizeServicesInActions(validator.validate(currentModel, nextModel, + ValidationOverrides.empty())); + exp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + act.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + assertThat(act, equalTo(exp)); + } + } + + static String STRING_FIELD = "field f1 type string { indexing: summary }"; + static String ATTRIBUTE_FIELD = "field f1 type string { indexing: attribute | summary }"; + static String ATTRIBUTE_CHANGE_MSG = "Field 'f1' changed: add attribute aspect"; + static String INT_FIELD = "field f1 type int { indexing: summary }"; + static String FIELD_TYPE_CHANGE_MSG = "Field 'f1' changed: data type: 'string' -> 'int'"; + private static List<ServiceInfo> FOO_SERVICE = Arrays.asList( + new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null")); + private static List<ServiceInfo> BAR_SERVICE = Arrays.asList( + new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null")); + + @Test + public void requireThatDocumentDatabaseChangeIsDiscovered() { + Fixture.newOneDocFixture(STRING_FIELD, ATTRIBUTE_FIELD). + assertValidation(newRestartAction("Document type 'd1': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE)); + } + + @Test + public void requireThatChangeInSeveralDocumentDatabasesAreDiscovered() { + Fixture.newTwoDocFixture(STRING_FIELD, ATTRIBUTE_FIELD). + assertValidation(Arrays.asList(newRestartAction("Document type 'd1': " + + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), + newRestartAction("Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE))); + } + + @Test + public void requireThatChangeInSeveralContentClustersAreDiscovered() { + Fixture.newTwoClusterFixture(STRING_FIELD, ATTRIBUTE_FIELD). + assertValidation(Arrays.asList(newRestartAction("Document type 'd1': " + + ATTRIBUTE_CHANGE_MSG, FOO_SERVICE), + newRestartAction("Document type 'd2': " + ATTRIBUTE_CHANGE_MSG, BAR_SERVICE))); + } + + @Test + public void requireThatAddingDocumentDatabaseIsOk() { + new Fixture(Fixture.newOneDocModel(STRING_FIELD), Fixture.newTwoDocModel(STRING_FIELD, STRING_FIELD)).assertValidation(); + } + + @Test + public void requireThatRemovingDocumentDatabaseIsOk() { + new Fixture(Fixture.newTwoDocModel(STRING_FIELD, STRING_FIELD), Fixture.newOneDocModel(STRING_FIELD)).assertValidation(); + } + + @Test + public void requireThatAddingContentClusterIsOk() { + new Fixture(Fixture.newOneDocModel(STRING_FIELD), Fixture.newTwoClusterModel(STRING_FIELD, STRING_FIELD)).assertValidation(); + } + + @Test + public void requireThatRemovingContentClusterIsOk() { + new Fixture(Fixture.newTwoClusterModel(STRING_FIELD, STRING_FIELD), Fixture.newOneDocModel(STRING_FIELD)).assertValidation(); + } + + @Test + public void requireThatChangingFieldTypeIsDiscovered() { + Fixture f = Fixture.newOneDocFixture(STRING_FIELD, INT_FIELD); + f.assertValidation(Arrays.asList(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1"))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java new file mode 100644 index 00000000000..5a358639af3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ConfigChangeRefeedAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class IndexingModeChangeValidatorTest { + + @Test + public void testChangingIndexMode() throws IOException, SAXException { + ValidationTester tester = new ValidationTester(); + + VespaModel oldModel = + tester.deploy(null, getServices(AbstractSearchCluster.IndexingMode.REALTIME), validationOverrides).getFirst(); + List<ConfigChangeAction> changeActions = + tester.deploy(oldModel, getServices(AbstractSearchCluster.IndexingMode.STREAMING), validationOverrides).getSecond(); + + assertRefeedChange(true, // allowed=true due to validation override + "Cluster 'default' changed indexing mode from 'indexed' to 'streaming'", + changeActions); + } + + private void assertRefeedChange(boolean allowed, String message, List<ConfigChangeAction> changeActions) { + List<ConfigChangeAction> refeedActions = changeActions.stream() + .filter(a -> a instanceof ConfigChangeRefeedAction) + .collect(Collectors.toList()); + assertEquals(1, refeedActions.size()); + assertEquals(allowed, refeedActions.get(0).allowed()); + assertTrue(refeedActions.get(0) instanceof ConfigChangeRefeedAction); + assertEquals("indexing-mode-change", ((ConfigChangeRefeedAction)refeedActions.get(0)).name()); + assertEquals(message, refeedActions.get(0).getMessage()); + } + + private static final String getServices(AbstractSearchCluster.IndexingMode indexingMode) { + return "<services version='1.0'>" + + " <content id='default' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <engine>" + + (indexingMode.equals(AbstractSearchCluster.IndexingMode.REALTIME) ? " <proton/>" : " <vds/>") + + " </engine>" + + " <documents>" + + " <document type='music' mode='" + + (indexingMode.equals(AbstractSearchCluster.IndexingMode.REALTIME) ? "index" : "streaming") + "'/>" + + " </documents>" + + " <nodes count='1'/>" + + " </content>" + + "</services>"; + } + + private static final String validationOverrides = + "<validation-overrides>\n" + + " <allow until='2000-01-14' comment='test override'>indexing-mode-change</allow>\n" + + "</validation-overrides>\n"; + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java new file mode 100644 index 00000000000..764a5e57783 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +public class StartupCommandChangeValidatorTest { + + @Test + public void requireThatDifferentStartupCommandIsDetected() { + MockRoot oldRoot = createRootWithChildren(new ServiceStub("evilservice", "rm -rf /")); + MockRoot newRoot = createRootWithChildren(new ServiceStub("evilservice", "rm -rf *")); + List<ConfigChangeAction> changes = getStartupCommandChanges(oldRoot, newRoot); + assertEquals(1, changes.size()); + assertEquals("evilservice", changes.get(0).getServices().get(0).getConfigId()); + } + + @Test + public void requireEmptyResultForEqualStartupCommand() { + MockRoot oldRoot = createRootWithChildren(new ServiceStub("evilservice", "./hax.sh")); + MockRoot newRoot = createRootWithChildren(new ServiceStub("evilservice", "./hax.sh")); + List<ConfigChangeAction> changes = getStartupCommandChanges(oldRoot, newRoot); + assertTrue(changes.isEmpty()); + } + + @Test + public void requireEmptyResultForDifferentServices() { + MockRoot oldRoot = createRootWithChildren(new ServiceStub("evilservice", "./hax.sh")); + MockRoot newRoot = createRootWithChildren(new ServiceStub("goodservice", "./hax.sh")); + List<ConfigChangeAction> changes = getStartupCommandChanges(oldRoot, newRoot); + assertTrue(changes.isEmpty()); + } + + private static List<ConfigChangeAction> getStartupCommandChanges( + AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { + StartupCommandChangeValidator validator = new StartupCommandChangeValidator(); + return validator.findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); + } + + private static MockRoot createRootWithChildren(AbstractConfigProducer<?>... children) { + MockRoot root = new MockRoot(); + Arrays.asList(children).forEach(root::addChild); + root.freezeModelTopology(); + return root; + } + + private static class ServiceStub extends AbstractService { + private final String startupCommand; + + public ServiceStub(String name, String startupCommand) { + super(name); + setHostResource(new HostResource(new Host(null, "localhost"))); + this.startupCommand = startupCommand; + } + + @Override + public String getStartupCommand() { + return startupCommand; + } + + @Override + public int getPortCount() { + return 0; + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java new file mode 100644 index 00000000000..8271e0409d7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import org.junit.Test; + +import java.util.List; + +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; + +public class AttributeChangeValidatorTest { + + private static class Fixture extends ContentClusterFixture { + AttributeChangeValidator validator; + + public Fixture(String currentSd, String nextSd) throws Exception { + super(currentSd, nextSd); + validator = new AttributeChangeValidator(currentDb().getDerivedConfiguration().getAttributeFields(), + currentDb().getDerivedConfiguration().getIndexSchema(), + currentDocType(), + nextDb().getDerivedConfiguration().getAttributeFields(), + nextDb().getDerivedConfiguration().getIndexSchema(), + nextDocType()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(); + } + + } + + @Test + public void requireThatAddingAttributeAspectRequireRestart() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary }", + "field f1 type string { indexing: attribute | summary }"); + f.assertValidation(newRestartAction( + "Field 'f1' changed: add attribute aspect")); + } + + @Test + public void requireThatRemovingAttributeAspectRequireRestart() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: attribute | summary }", + "field f1 type string { indexing: summary }"); + f.assertValidation(newRestartAction( + "Field 'f1' changed: remove attribute aspect")); + } + + @Test + public void requireThatAddingAttributeFieldIsOk() throws Exception { + Fixture f = new Fixture("", "field f1 type string { indexing: attribute | summary \n attribute: fast-search }"); + f.assertValidation(); + } + + @Test + public void requireThatRemovingAttributeFieldIsOk() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: attribute | summary }", ""); + f.assertValidation(); + } + + @Test + public void requireThatChangingFastSearchRequireRestart() throws Exception { + new Fixture("field f1 type string { indexing: attribute }", + "field f1 type string { indexing: attribute \n attribute: fast-search }"). + assertValidation(newRestartAction( + "Field 'f1' changed: add attribute 'fast-search'")); + } + + @Test + public void requireThatChangingFastAccessRequireRestart() throws Exception { + new Fixture("field f1 type string { indexing: attribute \n attribute: fast-access }", + "field f1 type string { indexing: attribute }"). + assertValidation(newRestartAction( + "Field 'f1' changed: remove attribute 'fast-access'")); + } + + @Test + public void requireThatChangingHugeRequireRestart() throws Exception { + new Fixture("field f1 type string { indexing: attribute }", + "field f1 type string { indexing: attribute \n attribute: huge }"). + assertValidation(newRestartAction( + "Field 'f1' changed: add attribute 'huge'")); + } + + @Test + public void requireThatChangingDensePostingListThresholdRequireRestart() throws Exception { + new Fixture( + "field f1 type predicate { indexing: attribute \n index { arity: 8 \n dense-posting-list-threshold: 0.2 } }", + "field f1 type predicate { indexing: attribute \n index { arity: 8 \n dense-posting-list-threshold: 0.4 } }"). + assertValidation(newRestartAction( + "Field 'f1' changed: change property 'dense-posting-list-threshold' from '0.2' to '0.4'")); + } + + @Test + public void requireThatRemovingAttributeAspectFromIndexFieldIsOk() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: index | attribute }", + "field f1 type string { indexing: index }"); + f.assertValidation(); + } + + @Test + public void requireThatRemovingAttributeAspectFromIndexAndSummaryFieldIsOk() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: index | attribute | summary }", + "field f1 type string { indexing: index | summary }"); + f.assertValidation(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java new file mode 100644 index 00000000000..086544ba6ef --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.search.DocumentDatabase; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Test fixture to setup current and next content clusters used for change validation. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public abstract class ContentClusterFixture { + protected ContentCluster currentCluster; + protected ContentCluster nextCluster; + + public ContentClusterFixture(String currentSd, String nextSd) throws Exception { + currentCluster = createCluster(currentSd); + nextCluster = createCluster(nextSd); + } + + private static ContentCluster createCluster(String sdContent) throws Exception { + return new ContentClusterBuilder().build( + ContentClusterUtils.createMockRoot( + Arrays.asList(new SearchDefinitionBuilder().content(sdContent).build()))); + } + + protected DocumentDatabase currentDb() { + return currentCluster.getSearch().getIndexed().getDocumentDbs().get(0); + } + + protected NewDocumentType currentDocType() { + return currentCluster.getDocumentDefinitions().get("test"); + } + + protected DocumentDatabase nextDb() { + return nextCluster.getSearch().getIndexed().getDocumentDbs().get(0); + } + + protected NewDocumentType nextDocType() { + return nextCluster.getDocumentDefinitions().get("test"); + } + + public void assertValidation() { + List<VespaConfigChangeAction> act = validate(); + assertThat(act.size(), is(0)); + } + + public void assertValidation(VespaConfigChangeAction exp) { + assertValidation(Arrays.asList(exp)); + } + + public void assertValidation(List<VespaConfigChangeAction> exp) { + List<VespaConfigChangeAction> act = validate(); + assertThat(act, equalTo(exp)); + } + + public abstract List<VespaConfigChangeAction> validate(); + +} + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java new file mode 100644 index 00000000000..4391878c5be --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; + +public class DocumentDatabaseChangeValidatorTest { + + private static class Fixture extends ContentClusterFixture { + DocumentDatabaseChangeValidator validator; + + public Fixture(String currentSd, String nextSd) throws Exception { + super(currentSd, nextSd); + validator = new DocumentDatabaseChangeValidator(currentDb(), currentDocType(), nextDb(), nextDocType()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(ValidationOverrides.empty()); + } + + } + + @Test + public void requireThatAttributeIndexAndDocumentTypeChangesAreDiscovered() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary } " + + "field f2 type string { indexing: summary } " + + "field f3 type int { indexing: summary }", + "field f1 type string { indexing: attribute | summary } " + + "field f2 type string { indexing: index | summary } " + + "field f3 type string { indexing: summary }"); + f.assertValidation(Arrays.asList( + newRestartAction("Field 'f1' changed: add attribute aspect"), + newRefeedAction("indexing-change", + ValidationOverrides.empty(), + "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + + "'{ input f2 | tokenize normalize stem:\"SHORTEST\" | index f2 | summary f2; }'"), + newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f3' changed: data type: 'int' -> 'string'"))); + } + + @Test + public void requireThatRemovingAttributeAspectFromIndexFieldIsOk() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: index | attribute }", + "field f1 type string { indexing: index }"); + f.assertValidation(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java new file mode 100644 index 00000000000..89f56470f1d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -0,0 +1,170 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; + +/** + * Test validation of changes between a current and next document type used in a document database. + * + * @author <a href="mailto:Tor.Egge@yahoo-inc.com">Tor Egge</a> + * @since 2014-11-25 + */ +public class DocumentTypeChangeValidatorTest { + + private static class Fixture extends ContentClusterFixture { + DocumentTypeChangeValidator validator; + + public Fixture(String currentSd, String nextSd) throws Exception { + super(currentSd, nextSd); + validator = new DocumentTypeChangeValidator(currentDocType(), nextDocType()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(ValidationOverrides.empty()); + } + + } + + @Test + public void requireThatFieldRemovalIsOK() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary }", + "field f2 type string { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatSameDataTypeIsOK() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary }", + "field f1 type string { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatDataTypeChangeIsNotOK() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary }", + "field f1 type int { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f1' changed: data type: 'string' -> 'int'")); + } + + @Test + public void requireThatAddingCollectionTypeIsNotOK() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary }", + "field f1 type array<string> { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f1' changed: data type: 'string' -> 'Array<string>'")); + } + + + @Test + public void requireThatSameNestedDataTypeIsOK() throws Exception { + Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", + "field f1 type array<string> { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatNestedDataTypeChangeIsNotOK() throws Exception { + Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", + "field f1 type array<int> { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'")); + } + + @Test + public void requireThatChangedCollectionTypeIsNotOK() throws Exception { + Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", + "field f1 type weightedset<string> { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'")); + } + + @Test + public void requireThatMultipleDataTypeChangesIsNotOK() throws Exception { + Fixture f = new Fixture("field f1 type string { indexing: summary } field f2 type int { indexing: summary }" , + "field f2 type string { indexing: summary } field f1 type int { indexing: summary }"); + f.assertValidation(Arrays.asList(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f1' changed: data type: 'string' -> 'int'"), + newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f2' changed: data type: 'int' -> 'string'"))); + } + + @Test + public void requireThatSameDataTypeInStructFieldIsOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }", + "struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatSameNestedDataTypeChangeInStructFieldIsOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }", + "struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatAddingFieldInStructFieldIsOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} } field f3 type s1 { indexing: summary }", + "struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatRemovingFieldInStructFieldIsOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }", + "struct s1 { field f1 type string {} } field f3 type s1 { indexing: summary }"); + f.assertValidation(); + } + + @Test + public void requireThatDataTypeChangeInStructFieldIsNotOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }", + "struct s1 { field f1 type int {} } field f2 type s1 { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'")); + } + + @Test + public void requireThatNestedDataTypeChangeInStructFieldIsNotOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }", + "struct s1 { field f1 type array<int> {} } field f2 type s1 { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'")); + } + + @Test + public void requireThatDataTypeChangeInNestedStructFieldIsNotOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }", + "struct s1 { field f1 type int {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'")); + } + + @Test + public void requireThatMultipleDataTypeChangesInStructFieldIsNotOK() throws Exception { + Fixture f = new Fixture("struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }", + "struct s1 { field f1 type int {} field f2 type string {} } field f3 type s1 { indexing: summary }"); + f.assertValidation(newRefeedAction("field-type-change", + ValidationOverrides.empty(), + "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java new file mode 100644 index 00000000000..0dea99c7b01 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class IndexingScriptChangeValidatorTest { + + private static class Fixture extends ContentClusterFixture { + IndexingScriptChangeValidator validator; + + public Fixture(String currentSd, String nextSd) throws Exception { + super(currentSd, nextSd); + validator = new IndexingScriptChangeValidator(currentDb().getDerivedConfiguration().getSearch(), + nextDb().getDerivedConfiguration().getSearch()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(ValidationOverrides.empty()); + } + } + + private static class ScriptFixture { + private final ScriptExpression currentScript; + private final ScriptExpression nextScript; + + public ScriptFixture(String currentScript, String nextScript) throws Exception { + this.currentScript = ScriptExpression.fromString(currentScript); + this.nextScript = ScriptExpression.fromString(nextScript); + } + + public boolean validate() { + return IndexingScriptChangeValidator.equalScripts(currentScript, nextScript); + } + } + + private static String FIELD = "field f1 type string"; + private static String FIELD_F2 = "field f2 type string"; + + private static VespaConfigChangeAction expectedAction(String changedMsg, String fromScript, String toScript) { + return expectedAction("f1", changedMsg, fromScript, toScript); + } + + private static VespaConfigChangeAction expectedAction(String field, String changedMsg, String fromScript, String toScript) { + return VespaRefeedAction.of("indexing-change", + ValidationOverrides.empty(), + "Field '" + field + "' changed: " + + (changedMsg.isEmpty() ? "" : changedMsg + ", ") + + "indexing script: '" + fromScript + "' -> '" + toScript + "'"); + } + + @Test + public void requireThatAddingIndexAspectRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: summary }", + FIELD + " { indexing: index | summary }"). + assertValidation(expectedAction("add index aspect", + "{ input f1 | summary f1; }", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1 | summary f1; }")); + } + + @Test + public void requireThatRemovingIndexAspectRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: index | summary }", + FIELD + " { indexing: summary }"). + assertValidation(expectedAction("remove index aspect", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1 | summary f1; }", + "{ input f1 | summary f1; }")); + } + + @Test + public void requireThatChangingStemmingRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: index }", + FIELD + " { indexing: index \n stemming: none }"). + assertValidation(expectedAction("stemming: 'shortest' -> 'none'", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", + "{ input f1 | tokenize normalize | index f1; }")); + } + + @Test + public void requireThatChangingNormalizingRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: index }", + FIELD + " { indexing: index \n normalizing: none }"). + assertValidation(expectedAction("normalizing: 'ACCENT' -> 'NONE'", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", + "{ input f1 | tokenize stem:\"SHORTEST\" | index f1; }")); + } + + @Test + public void requireThatChangingMatchingRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: index \n match: exact }", + FIELD + " { indexing: index \n match { gram \n gram-size: 3 } }"). + assertValidation(expectedAction("matching: 'exact' -> 'gram (size 3)', normalizing: 'LOWERCASE' -> 'CODEPOINT'", + "{ input f1 | exact | index f1; }", + "{ input f1 | ngram 3 | index f1; }")); + } + + @Test + public void requireThatSettingDynamicSummaryRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: summary }", + FIELD + " { indexing: summary \n summary: dynamic }"). + assertValidation(expectedAction("summary field 'f1' transform: 'none' -> 'dynamicteaser'", + "{ input f1 | summary f1; }", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | summary f1; }")); + } + + @Test + public void requireThatMultipleChangesRequireRefeed() throws Exception { + new Fixture(FIELD + " { indexing: index } " + FIELD_F2 + " { indexing: index }", + FIELD + " { indexing: index \n stemming: none } " + FIELD_F2 + " { indexing: index \n normalizing: none }"). + assertValidation(Arrays.asList(expectedAction("f1", "stemming: 'shortest' -> 'none'", + "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", + "{ input f1 | tokenize normalize | index f1; }"), + expectedAction("f2", "normalizing: 'ACCENT' -> 'NONE'", + "{ input f2 | tokenize normalize stem:\"SHORTEST\" | index f2; }", + "{ input f2 | tokenize stem:\"SHORTEST\" | index f2; }"))); + } + + @Test + public void requireThatAddingIndexFieldIsOk() throws Exception { + new Fixture("", "field f1 type string { indexing: index | summary }"). + assertValidation(); + } + + @Test + public void requireThatRemovingIndexFieldIsOk() throws Exception { + new Fixture("field f1 type string { indexing: index | summary }", ""). + assertValidation(); + } + + @Test + public void requireThatAddingFieldIsOk() throws Exception { + new Fixture("", FIELD + " { indexing: attribute | summary }"). + assertValidation(); + } + + @Test + public void requireThatAddingSummaryAspectIsOk() throws Exception { + new Fixture(FIELD + " { indexing: attribute }", + FIELD + " { indexing: attribute | summary }"). + assertValidation(); + } + + @Test + public void requireThatSettingDynamicSummaryOnIndexFieldIsOk() throws Exception { + new Fixture(FIELD + " { indexing: index | summary }", + FIELD + " { indexing: index | summary \n summary: dynamic }"). + assertValidation(); + } + + @Test + public void requireThatOutputExpressionsAreIgnoredInAdvancedScript() throws Exception { + assertTrue(new ScriptFixture("{ input foo | switch { case \"audio\": input bar | index; case \"video\": input baz | index; default: 0 | index; }; }", + "{ input foo | switch { case \"audio\": input bar | attribute; case \"video\": input baz | attribute; default: 0 | attribute; }; }"). + validate()); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java new file mode 100644 index 00000000000..db35e2cac1e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder; + +import com.yahoo.test.ArraytypesConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.ConfigDefinitionStore; +import com.yahoo.test.SimpletypesConfig; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class UserConfigBuilderTest { + + private final ConfigDefinitionStore configDefinitionStore = new ConfigDefinitionStore() { + @Override + public ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey) { return null; } + }; + + @Test + public void require_that_simple_config_is_resolved() throws ParserConfigurationException, IOException, SAXException { + Element configRoot = getDocument("<config name=\"simpletypes\">" + + " <intval>13</intval>" + + "</config>" + + "<config name=\"simpletypes\" version=\"1\">" + + " <stringval>foolio</stringval>" + + "</config>"); + UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); + assertFalse(map.isEmpty()); + ConfigDefinitionKey key = new ConfigDefinitionKey("simpletypes", "config"); + assertNotNull(map.get(key)); + SimpletypesConfig config = createConfig(SimpletypesConfig.class, map.get(key)); + assertThat(config.intval(), is(13)); + assertThat(config.stringval(), is("foolio")); + } + + public static <ConfigType extends ConfigInstance> ConfigType createConfig(Class<ConfigType> clazz, ConfigPayloadBuilder builder) { + return ConfigPayload.fromBuilder(builder).toInstance(clazz, ""); + } + + + @Test + public void require_that_arrays_config_is_resolved() throws ParserConfigurationException, IOException, SAXException { + Element configRoot = getDocument("<config name=\"arraytypes\">" + + " <intarr operation=\"append\">13</intarr>" + + " <intarr operation=\"append\">10</intarr>" + + " <intarr operation=\"append\">1337</intarr>" + + "</config>"); + UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); + assertFalse(map.isEmpty()); + ConfigDefinitionKey key = new ConfigDefinitionKey("arraytypes", "config"); + assertNotNull(map.get(key)); + ArraytypesConfig config = createConfig(ArraytypesConfig.class, map.get(key)); + assertThat(config.intarr().size(), is(3)); + assertThat(config.intarr(0), is(13)); + assertThat(config.intarr(1), is(10)); + assertThat(config.intarr(2), is(1337)); + } + + @Test + public void require_that_arrays_of_structs_are_resolved() throws ParserConfigurationException, IOException, SAXException { + Element configRoot = getDocument( + " <config name='vespa.configdefinition.specialtokens'>" + + " <tokenlist operation='append'>" + + " <name>default</name>" + + " <tokens operation='append'>" + + " <token>dvd+-r</token>" + + " </tokens>" + + " </tokenlist>" + + " </config>" + ); + assertArraysOfStructs(configRoot); + } + + private void assertArraysOfStructs(Element configRoot) { + UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); + assertFalse(map.isEmpty()); + ConfigDefinitionKey key = new ConfigDefinitionKey(SpecialtokensConfig.CONFIG_DEF_NAME, SpecialtokensConfig.CONFIG_DEF_NAMESPACE); + assertNotNull(map.get(key)); + SpecialtokensConfig config = createConfig(SpecialtokensConfig.class, map.get(key)); + assertThat(config.tokenlist().size(), is(1)); + assertThat(config.tokenlist().get(0).name(), is("default")); + assertThat(config.tokenlist().get(0).tokens().size(), is(1)); + assertThat(config.tokenlist().get(0).tokens().get(0).token(), is("dvd+-r")); + } + + @Test + public void no_exception_when_config_class_does_not_exist() throws ParserConfigurationException, IOException, SAXException { + Element configRoot = getDocument("<config name=\"unknown\">" + + " <foo>1</foo>" + + "</config>"); + UserConfigRepo repo = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); + ConfigPayloadBuilder builder = repo.get(new ConfigDefinitionKey("unknown", "config")); + assertNotNull(builder); + } + + private Element getDocument(String xml) throws ParserConfigurationException { + Reader xmlReader = new StringReader("<model>" + xml + "</model>"); + Document doc; + try { + doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + } catch (Exception e) { + throw new RuntimeException(); + } + return doc.getDocumentElement(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/Bug6068056Test.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/Bug6068056Test.java new file mode 100644 index 00000000000..beb408324c5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/Bug6068056Test.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Bug6068056Test { + private final static String HOSTS = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts>" + + " <host name=\"localhost\">" + + " <alias>node1</alias>" + + " </host>" + + "</hosts>"; + + private final static String SERVICES = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + + " <jdisc id=\"docproc\" version=\"1.0\">" + + " <search/>" + + " <document-processing/>" + + " <nodes>" + + " <node hostalias=\"node1\"/>" + + " </nodes>" + + " </jdisc>" + + + "<content version='1.0' id='music'>\n" + + " <redundancy>1</redundancy>\n" + + " <documents/>\n" + + " <group name='mygroup'>\n" + + " <node hostalias='node1' distribution-key='0'/>\n" + + " </group>\n" + + " <engine>\n" + + " <proton>\n" + + " <searchable-copies>1</searchable-copies>\n" + + " </proton>\n" + + " </engine>\n" + + " </content>" + + "</services>"; + + @Test(expected = RuntimeException.class) + public void testContainerClusterCalledDocproc() throws Exception { + VespaModelCreatorWithMockPkg creator = new VespaModelCreatorWithMockPkg(HOSTS, SERVICES); + creator.create(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java new file mode 100755 index 00000000000..98c27098c3d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -0,0 +1,214 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.admin.*; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class DomAdminV2BuilderTest extends DomBuilderTest { + + private static MockRoot root; + + @Before + public void prepareTest() throws Exception { + root = new MockRoot("root"); + } + + // Supported for backwards compatibility + private Element servicesConfigserver() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <configserver hostalias=\"mockhost\"/>" + + " <adminserver hostalias=\"mockhost\"/>" + + "</admin>").getDocumentElement(); + + } + + private Element servicesOverride() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"mockhost\"/>" + + " <config name=\"cloud.config.log.logd\">" + + " <logserver><host>foobar</host></logserver>" + + " </config>" + + "</admin>").getDocumentElement(); + + } + + private Element servicesConfigservers() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <configservers>" + + " <configserver hostalias=\"mockhost\"/>" + + " </configservers>" + + " <adminserver hostalias=\"mockhost\"/>" + + "</admin>").getDocumentElement(); + } + + private Element servicesYamas() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <configservers>" + + " <configserver hostalias=\"mockhost\"/>" + + " </configservers>" + + " <adminserver hostalias=\"mockhost\"/>" + + " <yamas systemname=\"foo\"/>" + + "</admin>").getDocumentElement(); + } + + private Element servicesNoYamas() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <configservers>" + + " <configserver hostalias=\"mockhost\"/>" + + " </configservers>" + + " <adminserver hostalias=\"mockhost\"/>" + + "</admin>").getDocumentElement(); + } + + private Element servicesAdminServerOnly() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"mockhost\"/>" + + "</admin>").getDocumentElement(); + } + + private Element servicesYamasIntervalOverride() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <configservers>" + + " <configserver hostalias=\"mockhost\"/>" + + " </configservers>" + + " <adminserver hostalias=\"mockhost\"/>" + + " <yamas systemname=\"foo\" interval=\"300\"/>" + + "</admin>").getDocumentElement(); + } + + private Element servicesMultitenantAdminOnly() { + return XML.getDocument( + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"mockhost\" />" + + "</admin>").getDocumentElement(); + } + + @Test + public void multitenant() { + List<ConfigServerSpec> configServerSpecs = Arrays.asList( + new Configserver.Spec("test1", 19070, 19071, 2181), + new Configserver.Spec("test2", 19070, 19071, 2181), + new Configserver.Spec("test3", 19070, 19071, 2181)); + Admin admin = buildAdmin(servicesMultitenantAdminOnly(), true, configServerSpecs); + assertThat(admin.getConfigservers().size(), is(3)); + assertThat(admin.getSlobroks().size(), is(1)); + assertThat(admin.getClusterControllerHosts().size(), is(1)); + assertNotNull(admin.getHostSystem().getHostByHostname("test1")); + for (Configserver configserver : admin.getConfigservers()) { + assertThat(configserver.getHostName(), is(not(admin.getClusterControllerHosts().get(0).getHost().getHostName()))); + for (Slobrok slobrok : admin.getSlobroks()) { + assertThat(slobrok.getHostName(), is(not(configserver.getHostName()))); + } + } + } + + /** + * Tests that configserver works (deprecated, but allowed in admin 2.0) + */ + @Test + public void adminWithConfigserverElement() { + Admin admin = buildAdmin(servicesConfigserver()); + assertThat(admin.getConfigservers().size(), is(1)); + } + + /** + * Tests that configservers/configserver works + */ + @Test + public void adminWithConfigserversElement() { + Admin admin = buildAdmin(servicesConfigservers()); + assertThat(admin.getConfigservers().size(), is(1)); + } + + @Test + public void basicYamasNoXml() { + Admin admin = buildAdmin(servicesNoYamas()); + Yamas y = admin.getYamas(); + assertThat(y.getClustername(), is("vespa")); + assertThat(y.getInterval(), is(1)); + } + + @Test + public void testAdminServerOnly() { + Admin admin = buildAdmin(servicesAdminServerOnly()); + assertEquals(1, admin.getSlobroks().size()); + } + + @Test + public void basicYamasXml() { + Admin admin = buildAdmin(servicesYamas()); + Yamas y = admin.getYamas(); + assertThat(y.getClustername(), is("foo")); + assertThat(y.getInterval(), is(1)); + } + + @Test + public void yamasWithIntervalOverride() { + Admin admin = buildAdmin(servicesYamasIntervalOverride()); + Yamas y = admin.getYamas(); + assertThat(y.getClustername(), is("foo")); + assertThat(y.getInterval(), is(5)); + } + + /** + * Test that illegal yamas interval throws exception + */ + @Test(expected = IllegalArgumentException.class) + public void yamasElementInvalid() { + Element servicesYamasIllegalInterval = XML.getDocument( + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"mockhost\"/>" + + " <yamas interval=\"5\"/>" + + "</admin>").getDocumentElement(); + Admin admin = buildAdmin(servicesYamasIllegalInterval); + } + + @Test + public void configOverridesCanBeUsedInAdmin() { + Admin admin = buildAdmin(servicesOverride()); + assertThat(admin.getUserConfigs().size(), is(1)); + LogdConfig.Builder logdBuilder = new LogdConfig.Builder(); + admin.addUserConfig(logdBuilder); + LogdConfig config = new LogdConfig(logdBuilder); + assertThat(config.logserver().host(), is("foobar")); + } + + private Admin buildAdmin(Element xml) { + return buildAdmin(xml, false, new ArrayList<>()); + } + + private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + final DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(root.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + Admin admin = domAdminBuilder.build(root, xml); + admin.addPerHostServices(root.getHostSystem().getHosts(), new DeployProperties.Builder().build()); + return admin; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java new file mode 100644 index 00000000000..c224c81fc34 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilderTest.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.vespa.model.container.component.Component; +import org.junit.Test; + +import static com.yahoo.collections.CollectionUtil.first; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + */ +public class DomComponentBuilderTest extends DomBuilderTest { + + @Test + public void ensureCorrectModel() { + Component<?, ?> handler = new DomComponentBuilder().doBuild(root, parse( + "<handler id='theId' class='theClass' bundle='theBundle' />")); + + BundleInstantiationSpecification instantiationSpecification = handler.model.bundleInstantiationSpec; + assertThat(instantiationSpecification.id.stringValue(), is("theId")); + assertThat(instantiationSpecification.classId.stringValue(), is("theClass")); + assertThat(instantiationSpecification.bundle.stringValue(), is("theBundle")); + } + + @Test + @SuppressWarnings("unchecked") + public void components_can_be_nested() { + Component<Component<?, ?>, ?> parent = new DomComponentBuilder().doBuild(root, parse( + "<component id='parent'>", + " <component id='child' />", + "</component>")); + + assertThat(parent.getGlobalComponentId(), is(ComponentId.fromString("parent"))); + Component<?, ?> child = first(parent.getChildren().values()); + assertNotNull(child); + + assertThat(child.getGlobalComponentId(), is(ComponentId.fromString("child@parent"))); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java new file mode 100644 index 00000000000..547b95357c5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -0,0 +1,326 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.slime.JsonFormat; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; + +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.*; +import java.util.ArrayList; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for the {@link com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder} class. + * + * @author gjoranv + * @author lulf + */ +public class DomConfigPayloadBuilderTest { + + @Test + public void testFunctionTest_DefaultValues() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new FileReader(new File("src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml"))); + ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<>())); + String expected = "" + + "{" + + "\"bool_val\":\"false\"," + + "\"int_val\":\"5\"," + + "\"long_val\":\"1234567890123\"," + + "\"double_val\":\"41.23\"," + + "\"string_val\":\"foo\"," + + "\"enum_val\":\"FOOBAR\"," + + "\"refval\":\":parent:\"," + + "\"fileVal\":\"vespa.log\"," + + "\"basicStruct\":{\"bar\":\"3\",\"intArr\":[\"10\"]}," + + "\"rootStruct\":{\"inner0\":{\"index\":\"11\"},\"inner1\":{\"index\":\"12\"}," + + "\"innerArr\":[{\"stringVal\":\"deep\"}]}," + + "\"boolarr\":[\"false\"]," + + "\"doublearr\":[\"2344\",\"123\"]," + + "\"stringarr\":[\"bar\"]," + + "\"enumarr\":[\"VALUES\"]," + + "\"myarray\":[{\"refval\":\":parent:\",\"fileVal\":\"command.com\",\"myStruct\":{\"a\":\"1\"},\"stringval\":[\"baah\",\"yikes\"],\"anotherarray\":[{\"foo\":\"7\"}]},{\"refval\":\":parent:\",\"fileVal\":\"display.sys\",\"myStruct\":{\"a\":\"-1\"},\"anotherarray\":[{\"foo\":\"1\"},{\"foo\":\"2\"}]}]" + + "}"; + assertPayload(expected, config); + } + + private void assertPayload(String expected, ConfigPayload payload) { + try { + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, payload.getSlime()); + assertThat(a.toString(), is(expected)); + } catch (Exception e) { + fail("Exception thrown when encoding slime: " + e.getMessage()); + } + + } + // Multi line strings are not tested in 'DefaultValues', so here it is. + @Test + public void verifyThatWhitespaceIsPreservedForStrings() throws Exception { + Element configRoot = getDocument(new FileReader(new File("src/test/cfg/admin/userconfigs/whitespace-test.xml"))); + ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"stringVal\":\" This is a string\\n that contains different kinds of whitespace \"}", config); + } + + @Test + public void put_to_leaf_map() throws Exception { + Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<config name=\"foobar\">" + + " <intmap>" + + " <item key=\"bar\">1338</item>" + + " <item key=\"foo\">1337</item>" + + " </intmap>" + + "</config>"); + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig), new ArrayList<String>())); + assertPayload("{\"intmap\":{\"bar\":\"1338\",\"foo\":\"1337\"}}", userConfig); + } + + @Test + public void put_to_inner_map() throws Exception { + Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<config name=\"foobar\">" + + " <innermap>" + + " <item key=\"bar\">" + + " <foo>baz</foo>" + + " </item>" + + " <item key=\"foo\">" + + " <foo>bar</foo>" + + " </item>" + + " </innermap>" + + "</config>"); + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig), new ArrayList<String>())); + assertPayload("{\"innermap\":{\"bar\":{\"foo\":\"baz\"},\"foo\":{\"foo\":\"bar\"}}}", userConfig); + } + + @Test + public void put_to_nested_map() throws Exception { + Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<config name=\"foobar\">" + + " <nestedmap>" + + " <item key=\"bar\">" + + " <inner>" + + " <item key=\"bar1\">30</item>" + + " <item key=\"bar2\">40</item>" + + " </inner>" + + " </item>" + + " <item key=\"foo\">" + + " <inner>" + + " <item key=\"foo1\">10</item>" + + " <item key=\"foo2\">20</item>" + + " </inner>" + + " </item>" + + " </nestedmap>" + + "</config>"); + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig), new ArrayList<String>())); + assertPayload("{\"nestedmap\":{" + + "\"bar\":{\"inner\":{\"bar1\":\"30\",\"bar2\":\"40\"}}," + + "\"foo\":{\"inner\":{\"foo1\":\"10\",\"foo2\":\"20\"}}}}", userConfig); + } + + @Test + public void append_to_leaf_array() throws Exception { + // Simulate user config from vespa-services.xml + Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<config name=\"function-test\">" + + " <intarr operation=\"append\">1</intarr>" + + " <intarr operation=\"append\">2</intarr>" + + "</config> "); + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig), new ArrayList<String>())); + assertPayload("{\"intarr\":[\"1\",\"2\"]}", userConfig); + } + + @Test + public void camel_case_via_dashes() throws Exception { + Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<config name=\"function-test\">" + + " <some-struct> <any-value>17</any-value> </some-struct>" + + "</config> "); + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(getDocument(xmlConfig), new ArrayList<String>())); + assertPayload("{\"someStruct\":{\"anyValue\":\"17\"}}", userConfig); + } + + // Verifies that an exception is thrown when the root element is not 'config'. + @Test + public void testFailWrongTagName() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<configs name=\"foo\"/>")); + try { + new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>()); + fail("Expected exception for wrong tag name."); + } catch (ConfigurationRuntimeException e) { + assertThat(e.getMessage(), + is("The root element must be 'config', but was 'configs'.")); + } + } + + // Verifies that an exception is thrown when the root element is not 'config'. + @Test + public void testFailNoNameAttribute() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<config/>")); + try { + new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>()); + fail("Expected exception for mismatch between def-name and xml name attribute."); + } catch (ConfigurationRuntimeException e) { + assertThat(e.getMessage(), + is("The 'config' element must have a 'name' attribute that matches the name of the config definition.")); + } + } + + @Test + public void testNamespace() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<config name=\"function-test\" namespace=\"config\">" + + "<int_val>1</int_val> +" + + "</config>")); + ConfigPayload config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"int_val\":\"1\"}", config); + + configRoot = getDocument(new StringReader("<config name=\"config.function-test\">" + + "<int_val>1</int_val> +" + + "</config>")); + config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"int_val\":\"1\"}", config); + + configRoot = getDocument(new StringReader("<config name=\"config.function_test\">" + + "<int_val>1</int_val> +" + + "</config>")); + config = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"int_val\":\"1\"}", config); + } + + @Test + public void testNameParsing() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<config name=\"function-test\" version=\"1\" namespace=\"config\">" + + "<int_val>1</int_val> +" + + "</config>")); + ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(configRoot); + assertThat(key.getName(), is("function-test")); + assertThat(key.getNamespace(), is("config")); + + configRoot = getDocument(new StringReader("<config name=\"function_test\" version=\"1\">" + + "<int_val>1</int_val> +" + + "</config>")); + key = DomConfigPayloadBuilder.parseConfigName(configRoot); + assertThat(key.getName(), is("function_test")); + assertThat(key.getNamespace(), is("config")); + + // Both namespace and name in name attribute + configRoot = getDocument(new StringReader("<config name=\"config.function-test\" version=\"1\">" + + "<int_val>1</int_val> +" + + "</config>")); + key = DomConfigPayloadBuilder.parseConfigName(configRoot); + assertThat(key.getName(), is("function-test")); + assertThat(key.getNamespace(), is("config")); + } + + @Test(expected = ConfigurationRuntimeException.class) + public void testNameParsingInvalidName() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<config name=\" function-test\" version=\"1\">" + + "<int_val>1</int_val> +" + + "</config>")); + DomConfigPayloadBuilder.parseConfigName(configRoot); + } + + @Test(expected = ConfigurationRuntimeException.class) + public void testNameParsingInvalidNamespace() throws FileNotFoundException, ParserConfigurationException { + Element configRoot = getDocument(new StringReader("<config name=\"function-test\" namespace=\"_foo\" version=\"1\">" + + "<int_val>1</int_val> +" + + "</config>")); + DomConfigPayloadBuilder.parseConfigName(configRoot); + } + + @Test + public void require_that_item_syntax_works_with_leaf() throws ParserConfigurationException { + Element configRoot = getDocument( + "<config name=\"arraytypes\" version=\"1\">" + + " <intarr>" + + " <item>13</item>" + + " <item>10</item>" + + " <item>1337</item>" + + " </intarr>" + + "</config>"); + + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"intarr\":[\"13\",\"10\",\"1337\"]}", userConfig); + } + + @Test + public void require_that_item_syntax_works_with_struct() throws ParserConfigurationException { + Element configRoot = getDocument( + "<config name=\"arraytypes\" version=\"1\">" + + " <lolarray>" + + " <item><foo>hei</foo><bar>hei2</bar></item>" + + " <item><foo>hoo</foo><bar>hoo2</bar></item>" + + " <item><foo>happ</foo><bar>happ2</bar></item>" + + " </lolarray>" + + "</config>"); + + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"lolarray\":[{\"foo\":\"hei\",\"bar\":\"hei2\"},{\"foo\":\"hoo\",\"bar\":\"hoo2\"},{\"foo\":\"happ\",\"bar\":\"happ2\"}]}", + userConfig); + } + + @Test + public void require_that_item_syntax_works_with_struct_array() throws ParserConfigurationException { + Element configRoot = getDocument( + "<config name=\"arraytypes\" version=\"1\">" + + " <lolarray>" + + " <item><fooarray><item>13</item></fooarray></item>" + + " <item><fooarray><item>10</item></fooarray></item>" + + " <item><fooarray><item>1337</item></fooarray></item>" + + " </lolarray>" + + "</config>"); + + ConfigPayload userConfig = ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>())); + assertPayload("{\"lolarray\":[{\"fooarray\":[\"13\"]},{\"fooarray\":[\"10\"]},{\"fooarray\":[\"1337\"]}]}", userConfig); + } + + @Test(expected = ConfigurationRuntimeException.class) + public void require_that_item_is_reserved_in_root() throws ParserConfigurationException { + Element configRoot = getDocument( + "<config name=\"arraytypes\" version=\"1\">" + + " <item>13</item>" + + "</config>"); + new DomConfigPayloadBuilder(null).build(configRoot, new ArrayList<String>()); + } + + @Test(expected=ConfigurationRuntimeException.class) + public void require_that_exceptions_are_issued() throws ParserConfigurationException, FileNotFoundException { + Element configRoot = getDocument( + "<config name=\"simpletypes\">" + + "<longval>invalid</longval>" + + "</config>"); + DefParser defParser = new DefParser("simpletypes", + new FileReader(new File("src/test/resources/configdefinitions/simpletypes.def"))); + ConfigDefinition def = ConfigDefinitionBuilder.createConfigDefinition(defParser.getTree()); + ConfigPayloadBuilder builder = new DomConfigPayloadBuilder(def).build(configRoot, new ArrayList<String>()); + //assertThat(builder.warnings().size(), is(1)); + } + + private Element getDocument(Reader xmlReader) throws ParserConfigurationException { + Document doc; + try { + doc = XmlHelper.getDocumentBuilder().parse(new InputSource(xmlReader)); + } catch (Exception e) { + throw new RuntimeException(); + } + return doc.getDocumentElement(); + } + + private Element getDocument(String xml) throws ParserConfigurationException { + Reader xmlReader = new StringReader(xml); + return getDocument(xmlReader); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilderTest.java new file mode 100644 index 00000000000..60d9fce767e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilderTest.java @@ -0,0 +1,817 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.config.GenericConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.engines.ProtonEngine; +import com.yahoo.vespa.model.content.engines.VDSEngine; +import com.yahoo.vespa.model.search.*; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.*; + +/** + * @author balder + */ +public class DomContentBuilderTest extends DomBuilderTest { + private ContentCluster createContent(String xml) throws Exception { + String combined = "" + + "<services>"+ + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost'/>" + + " </admin>" + + xml + + "</services>"; + + + VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withHosts(getHosts()) + .withServices(combined) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(); + + return m.getContentClusters().isEmpty() + ? null + : m.getContentClusters().values().iterator().next(); + } + private ContentCluster createContentWithBooksToo(String xml) throws Exception { + String combined = "" + + "<services>"+ + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost'/>" + + " </admin>" + + xml + + "</services>"; + + VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withHosts(getHosts()) + .withServices(combined) + .withSearchDefinitions(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, + MockApplicationPackage.BOOK_SEARCHDEFINITION)) + .build()) + .create(); + + return m.getContentClusters().isEmpty() + ? null + : m.getContentClusters().values().iterator().next(); + } + + private String getHosts() { + return "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts>" + + " <host name='node0'>" + + " <alias>mockhost</alias>" + + " </host>" + + " <host name='node1'>" + + " <alias>mockhost2</alias>" + + " </host>" + + " <host name='node2'>" + + " <alias>mockhost3</alias>" + + " </host>" + + "</hosts>"; + } + + private String getServices(String groupXml) { + return getConfigOverrideServices(groupXml, ""); + } + + private String getConfigOverrideServices(String groupXml, String documentOverrides) { + return "" + + "<services>"+ + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost'/>" + + " </admin>" + + " <jdisc version='1.0' id='qrc'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='mockhost' />" + + " </nodes>" + + " </jdisc>" + + " <content version='1.0' id='clu'>" + + " <documents>" + + " <document type='music' mode='index'>" + + documentOverrides + + " </document>" + + " </documents>" + + " <redundancy>3</redundancy>"+ + " <engine>" + + " <proton>" + + " <query-timeout>7.3</query-timeout>" + + " </proton>" + + " </engine>" + + " <group>"+ + groupXml + + " </group>"+ + " </content>" + + "</services>"; + } + + private String getBasicServices() { + return getServices("<node hostalias='mockhost' distribution-key='0'/>"); + } + + public static void assertServices(HostResource host, String [] services) { + String missing = ""; + + for (String s : services) { + if (host.getService(s) == null) { + missing += s + ","; + } + } + + String extra = ""; + for (Service s : host.getServices()) { + boolean found = false; + for (String n : services) { + if (n.equals(s.getServiceName())) { + found = true; + } + } + + if (!found) { + extra += s.getServiceName() + ","; + } + } + + assertEquals("Missing: Extra: ", "Missing: " + missing+ " Extra: " + extra); + + assertEquals(services.length, host.getServices().size()); + } + + @Test + public void handleSingleNonSearchPersistentDummy() throws Exception { + ContentCluster a = createContent( + "<content version =\"1.0\" id=\"a\">"+ + " <redundancy>3</redundancy>"+ + " <documents>" + + " <document type=\"music\" mode=\"store-only\"/>" + + " </documents>" + + " <engine>"+ + " <dummy/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + + ContentSearchCluster s = a.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertTrue(s.getClusters().isEmpty()); + + assertTrue(a.getPersistence() instanceof com.yahoo.vespa.model.content.engines.DummyPersistence.Factory); + } + + @Test + public void handleSingleNonSearchPersistentVds() throws Exception { + ContentCluster a = createContent( + "<content version =\"1.0\" id=\"a\">"+ + " <redundancy>3</redundancy>"+ + " <documents>" + + " <document type=\"music\" mode=\"store-only\"/>" + + " </documents>" + + " <engine>"+ + " <vds/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + + ContentSearchCluster s = a.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertTrue(s.getClusters().isEmpty()); + + assertTrue(a.getPersistence() instanceof VDSEngine.Factory); + + assertEquals(1, a.getStorageNodes().getChildren().size()); + } + + @Test + public void handleSingleNonSearchPersistentProton() throws Exception { + ContentCluster a = createContent( + "<content version =\"1.0\" id=\"a\">"+ + " <redundancy>3</redundancy>"+ + " <documents>" + + " <document type=\"music\" mode=\"store-only\"/>" + + " </documents>" + + " <engine>"+ + " <proton/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + + ContentSearchCluster s = a.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertTrue(s.getClusters().isEmpty()); + + assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); + + assertEquals(1, a.getStorageNodes().getChildren().size()); + } + + @Test + public void handleSingleNonSearchNonPersistentCluster() throws Exception { + ContentCluster a = createContent( + "<content version =\"1.0\" id=\"a\">"+ + " <redundancy>3</redundancy>"+ + " <documents>" + + " <document type=\"music\" mode=\"store-only\"/>" + + " </documents>" + + " <engine>"+ + " <vds/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + + ContentSearchCluster s = a.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertTrue(s.getClusters().isEmpty()); + assertNull(s.getIndexed()); + + assertNull(a.getRootGroup().getName()); + assertNull(a.getRootGroup().getIndex()); + assertTrue(a.getRootGroup().getSubgroups().isEmpty()); + assertEquals(1, a.getRootGroup().getNodes().size()); + assertEquals("node0", a.getRootGroup().getNodes().get(0).getHostName()); + + assertTrue(a.getPersistence() instanceof VDSEngine.Factory); + assertEquals(1, a.getStorageNodes().getChildren().size()); + assertEquals("a", a.getConfigId()); + } + + @Test + public void handleIndexedOnlyWithoutPersistence() throws Exception { + VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), getBasicServices())).create(); + + ContentCluster c = CollectionUtil.first(m.getContentClusters().values()); + ContentSearchCluster s = c.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertEquals(1, s.getClusters().size()); + assertNotNull(s.getIndexed()); + assertEquals("clu", s.getIndexed().getClusterName()); + assertEquals(7.3, s.getIndexed().getQueryTimeout(), 0.0); + + assertTrue(c.getPersistence() instanceof ProtonEngine.Factory); + assertEquals(1, c.getStorageNodes().getChildren().size()); + assertEquals("clu", c.getConfigId()); + //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); // This is how it should look like in an ideal world. + assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. + assertEquals(1, c.getRoot().getHostSystem().getHosts().size()); + HostResource h = c.getRoot().getHostSystem().getHost("mockhost"); + String [] expectedServices = {"logd", "configproxy","config-sentinel", "qrserver", "storagenode", "searchnode", "distributor", "topleveldispatch", "transactionlogserver"}; +// TODO assertServices(h, expectedServices); + assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); + assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); + assertEquals("clu/distributor/0", h.getService("distributor").getConfigId()); + assertEquals("clu/search/cluster.clu/tlds/qrc.0.tld.0", h.getService("topleveldispatch").getConfigId()); + //assertEquals("tcp/node0:19104", h.getService("topleveldispatch").getConfig("partitions", "").innerArray("dataset").value("0").innerArray("engine").value("0").getString("name_and_port")); + PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) + m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); + assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); + } + + @Test + public void testConfigIdLookup() throws Exception { + VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), getBasicServices())).create(); + + PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) + m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); + assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); + } + + @Test + public void testMultipleSearchNodesOnSameHost() throws Exception { + String services = getServices("<node hostalias='mockhost' distribution-key='0'/>" + + "<node hostalias='mockhost' distribution-key='1'/>"); + VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), services)).create(); + PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) + m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); + assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); + IndexedSearchCluster sc = m.getContentClusters().get("clu").getSearch().getIndexed(); + assertEquals(2, sc.getSearchNodeCount()); + assertTrue(sc.getSearchNode(0).getPersistenceProviderRpcPort() >= 19100); + assertTrue(sc.getSearchNode(0).getPersistenceProviderRpcPort() != sc.getSearchNode(1).getPersistenceProviderRpcPort()); + } + + @Test + public void handleStreamingOnlyWithoutPersistence() throws Exception + { + final String musicClusterId = "music-cluster-id"; + + ContentCluster cluster = createContent( + "<content version='1.0' id='" + musicClusterId + "'>" + + " <redundancy>3</redundancy>"+ + " <documents>"+ + " <document type='music' mode='streaming'/>"+ + " </documents>"+ + " <engine>"+ + " <vds/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + ContentSearchCluster s; + + s = cluster.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertEquals(1, s.getClusters().size()); + assertNull(s.getIndexed()); + AbstractSearchCluster sc = s.getClusters().get(musicClusterId + ".music"); + assertEquals(musicClusterId + ".music", sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster)sc).getStorageRouteSpec()); + + assertTrue(cluster.getPersistence() instanceof VDSEngine.Factory); + assertEquals(1, cluster.getStorageNodes().getChildren().size()); + + assertEquals(musicClusterId, cluster.getConfigId()); + //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); + assertEquals(musicClusterId + "/storage/0", cluster.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. + assertEquals(1, cluster.getRoot().getHostSystem().getHosts().size()); + HostResource h = cluster.getRoot().getHostSystem().getHost("mockhost"); + String [] expectedServices = { + "logd", "configproxy", + "config-sentinel", "configserver", "logserver", + "slobrok", "container-clustercontroller", + "filedistributorservice", "storagenode", "distributor" + }; + assertServices(h, expectedServices); + + assertEquals(musicClusterId + "/storage/0", h.getService("storagenode").getConfigId()); + + /* Not yet + assertNotNull(h.getService("qrserver")); + assertNotNull(h.getService("topleveldisptach")); + assertNotNull(h.getService("docproc")); + */ + + } + + @Test + public void requireThatContentStreamingHandlesMultipleSearchDefinitions() throws Exception + { + final String musicClusterId = "music-cluster-id"; + + ContentCluster cluster = createContentWithBooksToo( + "<content version='1.0' id='" + musicClusterId + "'>" + + " <redundancy>3</redundancy>"+ + " <documents>"+ + " <document type='music' mode='streaming'/>"+ + " <document type='book' mode='streaming'/>"+ + " </documents>"+ + " <engine>"+ + " <vds/>"+ + " </engine>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + ContentSearchCluster s; + + s = cluster.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertEquals(2, s.getClusters().size()); + assertNull(s.getIndexed()); + { + String id = musicClusterId + ".book"; + AbstractSearchCluster sc = s.getClusters().get(id); + assertEquals(id, sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec()); + } + { + String id = musicClusterId + ".music"; + AbstractSearchCluster sc = s.getClusters().get(id); + assertEquals(id, sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec()); + } + + assertTrue(cluster.getPersistence() instanceof VDSEngine.Factory); + assertEquals(1, cluster.getStorageNodes().getChildren().size()); + + assertEquals(musicClusterId, cluster.getConfigId()); + } + + @Test + public void handleIndexedWithoutPersistence() throws Exception + { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>3</redundancy>"+ + " <documents>"+ + " <document type='music' mode='index'/>"+ + " </documents>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertEquals(1, s.getClusters().size()); + assertNotNull(s.getIndexed()); + assertEquals("b", s.getIndexed().getClusterName()); + + assertTrue(b.getPersistence() instanceof ProtonEngine.Factory); + assertEquals(1, b.getStorageNodes().getChildren().size()); + + assertEquals("b", b.getConfigId()); + //assertEquals("content/a/0", a.getRootGroup().getNodes().get(0).getConfigId()); + assertEquals("b/storage/0", b.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. + assertEquals(1, b.getRoot().getHostSystem().getHosts().size()); + HostResource h = b.getRoot().getHostSystem().getHost("mockhost"); + assertEquals("b/storage/0", h.getService("storagenode").getConfigId()); + } + + @Test + public void canConfigureMmapNoCoreLimit() throws Exception { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group mmap-nocore-limit=\"200000\">" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + + " <node hostalias=\"mockhost\" distribution-key=\"1\" />" + + " </group>" + + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(2, b.getStorageNodes().getChildren().size()); + assertTrue(b.getRootGroup().getMmapNoCoreLimit().isPresent()); + assertEquals(200000, b.getRootGroup().getMmapNoCoreLimit().get().longValue()); + + assertThat(s.getSearchNodes().size(), is(2)); + assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); + assertEquals(200000, s.getSearchNodes().get(1).getMMapNoCoreLimit()); + assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); + assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + } + + @Test + public void canConfigureMmapNoCoreLimitPerHost() throws Exception { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\" mmap-nocore-limit=\"200000\"/>" + + " <node hostalias=\"mockhost\" distribution-key=\"1\" />" + + " </group>" + + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(2, b.getStorageNodes().getChildren().size()); + assertFalse(b.getRootGroup().getMmapNoCoreLimit().isPresent()); + + assertThat(s.getSearchNodes().size(), is(2)); + assertEquals(200000, s.getSearchNodes().get(0).getMMapNoCoreLimit()); + assertEquals(-1, s.getSearchNodes().get(1).getMMapNoCoreLimit()); + assertEquals("VESPA_MMAP_NOCORE_LIMIT=200000 ", s.getSearchNodes().get(0).getMMapNoCoreEnvVariable()); + assertEquals("", s.getSearchNodes().get(1).getMMapNoCoreEnvVariable()); + } + + @Test + public void canConfigureCpuAffinity() throws Exception + { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>2</redundancy>"+ + " <documents>"+ + " <document type='music' mode='index'/>"+ + " </documents>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\" cpu-socket=\"0\" />"+ + " <node hostalias=\"mockhost\" distribution-key=\"1\" cpu-socket=\"1\" />"+ + " </group>"+ + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(2, b.getStorageNodes().getChildren().size()); + assertTrue(b.getStorageNodes().getChildren().get("0").getAffinity().isPresent()); + assertThat(b.getStorageNodes().getChildren().get("0").getAffinity().get().cpuSocket(), is(0)); + assertTrue(b.getStorageNodes().getChildren().get("1").getAffinity().isPresent()); + assertThat(b.getStorageNodes().getChildren().get("1").getAffinity().get().cpuSocket(), is(1)); + + assertThat(s.getSearchNodes().size(), is(2)); + assertTrue(s.getSearchNodes().get(0).getAffinity().isPresent()); + assertThat(s.getSearchNodes().get(0).getAffinity().get().cpuSocket(), is(0)); + assertTrue(s.getSearchNodes().get(1).getAffinity().isPresent()); + assertThat(s.getSearchNodes().get(1).getAffinity().get().cpuSocket(), is(1)); + } + + @Test + public void canConfigureCpuAffinityAutomatically() throws Exception + { + ContentCluster b = createContent( + "<content version =\"1.0\" id=\"b\">" + + " <redundancy>2</redundancy>"+ + " <documents>"+ + " <document type='music' mode='index'/>"+ + " </documents>"+ + " <group cpu-socket-affinity=\"true\">"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\" />"+ + " <node hostalias=\"mockhost\" distribution-key=\"1\" />"+ + " <node hostalias=\"mockhost\" distribution-key=\"2\" />"+ + " <node hostalias=\"mockhost2\" distribution-key=\"3\" />"+ + " <node hostalias=\"mockhost2\" distribution-key=\"4\" />"+ + " <node hostalias=\"mockhost3\" distribution-key=\"5\" />"+ + " </group>"+ + "</content>"); + ContentSearchCluster s; + + s = b.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertNotNull(s.getIndexed()); + assertEquals(6, b.getStorageNodes().getChildren().size()); + assertTrue(b.getRootGroup().useCpuSocketAffinity()); + + assertThat(s.getSearchNodes().size(), is(6)); + assertTrue(s.getSearchNodes().get(0).getAffinity().isPresent()); + assertTrue(s.getSearchNodes().get(1).getAffinity().isPresent()); + assertTrue(s.getSearchNodes().get(2).getAffinity().isPresent()); + assertTrue(s.getSearchNodes().get(3).getAffinity().isPresent()); + assertTrue(s.getSearchNodes().get(4).getAffinity().isPresent()); + assertTrue(s.getSearchNodes().get(5).getAffinity().isPresent()); + assertThat(s.getSearchNodes().get(0).getAffinity().get().cpuSocket(),is (0)); + assertThat(s.getSearchNodes().get(1).getAffinity().get().cpuSocket(),is (1)); + assertThat(s.getSearchNodes().get(2).getAffinity().get().cpuSocket(),is (2)); + assertThat(s.getSearchNodes().get(3).getAffinity().get().cpuSocket(),is (0)); + assertThat(s.getSearchNodes().get(4).getAffinity().get().cpuSocket(),is (1)); + assertThat(s.getSearchNodes().get(5).getAffinity().get().cpuSocket(),is (0)); + + // TODO: Only needed for the search nodes anyway? + assertFalse(b.getStorageNodes().getChildren().get("0").getAffinity().isPresent()); + assertFalse(b.getStorageNodes().getChildren().get("1").getAffinity().isPresent()); + assertFalse(b.getStorageNodes().getChildren().get("2").getAffinity().isPresent()); + assertFalse(b.getStorageNodes().getChildren().get("3").getAffinity().isPresent()); + assertFalse(b.getStorageNodes().getChildren().get("4").getAffinity().isPresent()); + assertFalse(b.getStorageNodes().getChildren().get("5").getAffinity().isPresent()); + //assertThat(b.getStorageNodes().getChildren().get("0").getAffinity().get().cpuSocket(), is(0)); + //assertThat(b.getStorageNodes().getChildren().get("1").getAffinity().get().cpuSocket(), is(1)); + //assertThat(b.getStorageNodes().getChildren().get("2").getAffinity().get().cpuSocket(), is(2)); + //assertThat(b.getStorageNodes().getChildren().get("3").getAffinity().get().cpuSocket(), is(0)); + //assertThat(b.getStorageNodes().getChildren().get("4").getAffinity().get().cpuSocket(), is(1)); + //assertThat(b.getStorageNodes().getChildren().get("5").getAffinity().get().cpuSocket(), is(0)); + + } + + @Test + public void requireBug5357273() throws Exception { + try { + createContent( + " <content version='1.0' id='storage'>\n" + + " <redundancy>3</redundancy>\n" + + " <documents>"+ + " <document type='music' mode='index'/>"+ + " </documents>" + + " <group>\n" + + " <node hostalias='mockhost' distribution-key='0' />\n" + + " </group>\n" + + " <engine>\n" + + " <vds/>\n" + + " </engine>\n" + + " </content>\n"); + + assertFalse(true); + } catch (Exception e) { + e.printStackTrace(); + assertEquals("Persistence engine does not allow for indexed search. Please use <proton> as your engine.", e.getMessage()); + } + } + + @Test + public void handleProtonTuning() throws Exception{ + ContentCluster a = createContent( + "<content version =\"1.0\" id=\"a\">" + + " <redundancy>3</redundancy>" + + " <engine>" + + " <proton>" + + " <tuning>" + + " <searchnode>" + + " <summary>" + + " <store>" + + " <cache>" + + " <maxsize>8192</maxsize>" + + " <maxentries>32</maxentries>" + + " <compression>" + + " <type>lz4</type>" + + " <level>8</level>" + + " </compression>" + + " </cache>" + + " </store>" + + " <io>" + + " <read>directio</read>" + + " </io>" + + " </summary>" + + " </searchnode>" + + " </tuning>" + + " </proton>" + + " </engine>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>" + + " </group>" + + "</content>" + ); + + assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + a.getSearch().getConfig(pb); + List<String> serialize = ConfigInstance.serialize(new ProtonConfig(pb)); + String cfg = StringUtilities.implode(serialize.toArray(new String[serialize.size()]), "\n"); + assertThat(cfg, containsString("summary.cache.maxbytes 8192")); + assertThat(cfg, containsString("summary.cache.initialentries 32")); + assertThat(cfg, containsString("summary.cache.compression.level 8")); + assertThat(cfg, containsString("summary.cache.compression.type LZ4")); + assertThat(cfg, containsString("summary.read.io DIRECTIO")); + } + + @Test + public void requireThatUserConfigCanBeSpecifiedForASearchDefinition() throws Exception { + String services = getConfigOverrideServices( + "<node hostalias='mockhost' distribution-key='0'/>", + " <config name='mynamespace.myconfig'>" + + " <myfield>myvalue</myfield>" + + " </config>" + ); + + VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), services)).create(); + String configId = "clu/search/cluster.clu/music"; + { + GenericConfig.GenericConfigBuilder builder = + new GenericConfig.GenericConfigBuilder(new ConfigDefinitionKey("myconfig", "mynamespace"), new ConfigPayloadBuilder()); + m.getConfig(builder, configId); + assertEquals(builder.getPayload().getSlime().get().field("myfield").asString(), "myvalue"); + } + } + + @Test + public void requireOneTldPerSearchContainer() throws Exception { + ContentCluster content = createContent( + " <content version='1.0' id='storage'>\n" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>\n" + + " <node hostalias='mockhost' distribution-key='0' />\n" + + " </group>\n" + + " </content>\n" + + " <jdisc version='1.0' id='qrc'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='mockhost' />" + + " </nodes>" + + " </jdisc>" + + " <jdisc version='1.0' id='qrc2'>" + + " <http>" + + " <server id ='server1' port='5000' />" + + " </http>" + + " <search/>" + + " <nodes>" + + " <node hostalias='mockhost' />" + + " <node hostalias='mockhost2' />" + + " </nodes>" + + " </jdisc>" + + ); + List<Dispatch> tlds = content.getSearch().getIndexed().getTLDs(); + + assertThat(tlds.get(0).getHostname(), is("node0")); + assertThat(tlds.get(1).getHostname(), is("node0")); + assertThat(tlds.get(2).getHostname(), is("node1")); + + assertThat(tlds.size(), is(3)); + } + + @Test + @Ignore + public void ensureOverrideAppendedOnlyOnce() throws Exception { + ContentCluster content = createContent( + "<content version='1.0' id='search'>" + + " <config name=\"vespa.config.search.core.proton\">" + + " <numthreadspersearch>1</numthreadspersearch>" + + " <search>" + + " <mmap>" + + " <options><item>POPULATE</item></options>" + + " </mmap>" + + " </search>" + + " </config>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias='mockhost' distribution-key='0'/>" + + " </group>" + + "</content>"); + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + content.getSearch().getIndexed().getSearchNode(0).cascadeConfig(builder); + content.getSearch().getIndexed().getSearchNode(0).addUserConfig(builder); + ProtonConfig config = new ProtonConfig(builder); + assertThat(config.search().mmap().options().size(), is(1)); + assertThat(config.search().mmap().options(0), is(ProtonConfig.Search.Mmap.Options.POPULATE)); + } + + @Test + public void ensurePruneRemovedDocumentsAgeForHostedVespa() throws Exception { + { + ContentCluster contentNonHosted = createContent("<content version='1.0' id='search'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes>" + + " <node hostalias='mockhost' distribution-key='0'/>" + + " </nodes>" + + "</content>"); + ProtonConfig configNonHosted = getProtonConfig(contentNonHosted); + ProtonConfig defaultConfig = new ProtonConfig(new ProtonConfig.Builder()); + assertEquals(defaultConfig.pruneremoveddocumentsage(), configNonHosted.pruneremoveddocumentsage(), 0.001); + } + + { + String hostedXml = "<services>" + + "<content version='1.0' id='search'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='1'/>" + + "</content>" + + "</services>"; + + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties( + new DeployProperties.Builder() + .hostedVespa(true) + .build()); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(hostedXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(349260.0, config.pruneremoveddocumentsage(), 0.001); + } + } + + private ProtonConfig getProtonConfig(ContentCluster content) { + ProtonConfig.Builder configBuilder = new ProtonConfig.Builder(); + content.getSearch().getIndexed().getSearchNode(0).cascadeConfig(configBuilder); + content.getSearch().getIndexed().getSearchNode(0).addUserConfig(configBuilder); + + return new ProtonConfig(configBuilder); + } + + ApplicationPackage createAppWithMusic(String hosts, String services) { + return new MockApplicationPackage.Builder() + .withHosts(hosts) + .withServices(services) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java new file mode 100644 index 00000000000..6a9450b3d4e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -0,0 +1,228 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.model.search.Tuning; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Arrays; + +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DomSearchTuningBuilderTest extends DomBuilderTest { + + private static final double DELTA = 0.000001; + + private static Element parseXml(String... xmlLines) { + return parse("<tuning>", + "<searchnode>", + CollectionUtil.mkString(Arrays.asList(xmlLines), "\n"), + "</searchnode>", + "</tuning>"); + } + + private Tuning newTuning(String xml) { + return createTuning(parse(xml)); + } + + private Tuning createTuning(Element xml) { + DomSearchTuningBuilder b = new DomSearchTuningBuilder(); + return b.build(root, xml); + } + + String getProtonCfg(Tuning tuning) { + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + tuning.getConfig(pb); + return StringUtilities.implode(ConfigInstance.serialize(new ProtonConfig(pb)).toArray(new String[0]), "\n"); + } + + @Test + public void requireThatNullDispatchIsSafe() { + Tuning tuning = newTuning("<tuning />"); + assertNull(tuning.dispatch); + } + + @Test + public void requireThatEmptyDispatchIsSafe() { + Tuning tuning = newTuning("<tuning><dispatch/></tuning>"); + Tuning.Dispatch dispatch = tuning.dispatch; + assertNotNull(dispatch); + assertNull(dispatch.maxHitsPerPartition); + } + + @Test + public void requireThatDispatchSettingsAreParsed() { + Tuning tuning = createTuning(parse("<tuning>" + + " <dispatch>" + + " <max-hits-per-partition>69</max-hits-per-partition>" + + " </dispatch>" + + "</tuning>")); + Tuning.Dispatch dispatch = tuning.dispatch; + assertNotNull(dispatch); + assertNotNull(dispatch.maxHitsPerPartition); + assertEquals(69, dispatch.maxHitsPerPartition.intValue()); + } + + @Test + public void requireThatWeCanParseRequestThreadsTag() { + Tuning t = createTuning(parseXml("<requestthreads>", + "<search>123</search>", + "<persearch>34</persearch>", + "<summary>456</summary>", + "</requestthreads>")); + assertEquals(123, t.searchNode.threads.numSearchThreads.longValue()); + assertEquals(456, t.searchNode.threads.numSummaryThreads.longValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("numsearcherthreads 123")); + assertThat(cfg, containsString("numthreadspersearch 34")); + assertThat(cfg, containsString("numsummarythreads 456")); + } + + @Test + public void requireThatWeCanParseFlushStrategyTag() { + Tuning t = createTuning(parseXml("<flushstrategy>","<native>", + "<total>", + "<maxmemorygain>900</maxmemorygain>", + "<diskbloatfactor>8.7</diskbloatfactor>", + "</total>", + "<component>", + "<maxmemorygain>600</maxmemorygain>", + "<diskbloatfactor>5.4</diskbloatfactor>", + "<maxage>300</maxage>", + "</component>", + "<transactionlog>", + "<maxentries>200</maxentries>", + "<maxsize>1024</maxsize>", + "</transactionlog>", + "</native>","</flushstrategy>")); + assertEquals(900, t.searchNode.strategy.totalMaxMemoryGain.longValue()); + assertEquals(8.7, t.searchNode.strategy.totalDiskBloatFactor.doubleValue(), DELTA); + assertEquals(600, t.searchNode.strategy.componentMaxMemoryGain.longValue()); + assertEquals(5.4, t.searchNode.strategy.componentDiskBloatFactor.doubleValue(), DELTA); + assertEquals(300, t.searchNode.strategy.componentMaxage.doubleValue(), DELTA); + assertEquals(200, t.searchNode.strategy.transactionLogMaxEntries.longValue()); + assertEquals(1024, t.searchNode.strategy.transactionLogMaxSize.longValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("flush.memory.maxmemory 900")); + assertThat(cfg, containsString("flush.memory.diskbloatfactor 8.7")); + assertThat(cfg, containsString("flush.memory.each.maxmemory 600")); + assertThat(cfg, containsString("flush.memory.each.diskbloatfactor 5.4")); + assertThat(cfg, containsString("flush.memory.maxage.time 300")); + assertThat(cfg, containsString("flush.memory.maxage.serial 200")); + assertThat(cfg, containsString("flush.memory.maxtlssize 1024")); + } + + @Test + public void requireThatWeCanParseResizingTag() { + Tuning t = createTuning(parseXml("<resizing>", + "<initialdocumentcount>128</initialdocumentcount>", + "</resizing>")); + assertEquals(128, t.searchNode.resizing.initialDocumentCount.intValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("grow.initial 128")); + } + + @Test + public void requireThatWeCanParseIndexTag() { + Tuning t = createTuning(parseXml("<index>", "<io>", + "<write>directio</write>", + "<read>normal</read>", + "<search>mmap</search>", + "</io>", "</index>")); + assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.index.io.write); + assertEquals(Tuning.SearchNode.IoType.NORMAL, t.searchNode.index.io.read); + assertEquals(Tuning.SearchNode.IoType.MMAP, t.searchNode.index.io.search); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("indexing.write.io DIRECTIO")); + assertThat(cfg, containsString("indexing.read.io NORMAL")); + assertThat(cfg, containsString("search.io MMAP")); + } + + @Test + public void requireThatWeCanParseAttributeTag() { + Tuning t = createTuning(parseXml("<attribute>", "<io>", + "<write>directio</write>", + "</io>", "</attribute>")); + assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.attribute.io.write); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("attribute.write.io DIRECTIO")); + } + + @Test + public void requireThatWeCanParseSummaryTag() { + Tuning t = createTuning(parseXml("<summary>", + "<io>", + "<write>directio</write>", + "<read>directio</read>", + "</io>", + "<store>", + "<cache>", + "<maxsize>128</maxsize>", + "<maxentries>64</maxentries>", + "<compression>", + "<type>none</type>", + "<level>3</level>", + "</compression>", + "</cache>", + "<logstore>", + "<maxfilesize>512</maxfilesize>", + "<maxdiskbloatfactor>1.4</maxdiskbloatfactor>", + "<minfilesizefactor>0.3</minfilesizefactor>", + "<numthreads>7</numthreads>", + "<chunk>", + "<maxsize>256</maxsize>", + "<maxentries>32</maxentries>", + "<compression>", + "<type>lz4</type>", + "<level>5</level>", + "</compression>", + "</chunk>", + "</logstore>", + "</store>", + "</summary>")); + assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.summary.io.write); + assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.summary.io.read); + assertEquals(128, t.searchNode.summary.store.cache.maxSize.longValue()); + assertEquals(64, t.searchNode.summary.store.cache.maxEntries.longValue()); + assertEquals(Tuning.SearchNode.Summary.Store.Compression.Type.NONE, + t.searchNode.summary.store.cache.compression.type); + assertEquals(3, t.searchNode.summary.store.cache.compression.level.intValue()); + assertEquals(512, t.searchNode.summary.store.logStore.maxFileSize.longValue()); + assertEquals(1.4, t.searchNode.summary.store.logStore.maxDiskBloatFactor, DELTA); + assertEquals(0.3, t.searchNode.summary.store.logStore.minFileSizeFactor, DELTA); + assertEquals(7, t.searchNode.summary.store.logStore.numThreads.intValue()); + assertEquals(256, t.searchNode.summary.store.logStore.chunk.maxSize.intValue()); + assertEquals(32, t.searchNode.summary.store.logStore.chunk.maxEntries.intValue()); + assertEquals(Tuning.SearchNode.Summary.Store.Compression.Type.LZ4, + t.searchNode.summary.store.logStore.chunk.compression.type); + assertEquals(5, t.searchNode.summary.store.logStore.chunk.compression.level.intValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("summary.write.io DIRECTIO")); + assertThat(cfg, containsString("summary.read.io DIRECTIO")); + assertThat(cfg, containsString("summary.cache.maxbytes 128")); + assertThat(cfg, containsString("summary.cache.initialentries 64")); + assertThat(cfg, containsString("summary.cache.compression.type NONE")); + assertThat(cfg, containsString("summary.cache.compression.level 3")); + assertThat(cfg, containsString("summary.log.maxfilesize 512")); + assertThat(cfg, containsString("summary.log.maxdiskbloatfactor 1.4")); + assertThat(cfg, containsString("summary.log.minfilesizefactor 0.3")); + assertThat(cfg, containsString("summary.log.chunk.maxbytes 256")); + assertThat(cfg, containsString("summary.log.chunk.maxentries 32")); + assertThat(cfg, containsString("summary.log.chunk.compression.type LZ4")); + assertThat(cfg, containsString("summary.log.chunk.compression.level 5")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java new file mode 100644 index 00000000000..8239cb9cde0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilderTest.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class LegacyConfigModelBuilderTest { + @Test + public void testThatProducerIsInserted() { + String services = "<foo><config name=\"bar\"><key>value</key></config></foo>"; + ModelBuilder builder = new ModelBuilder(); + Model model = builder.build(DeployState.createTestState(new MockApplicationPackage.Builder().withServices(services).build()), + null, new MockRoot(), XML.getDocument(services).getDocumentElement()); + assertThat(model.getContext().getParentProducer().getUserConfigs().size(), is(1)); + } + + public static class Model extends ConfigModel { + + private final ConfigModelContext context; + + /** + * Constructs a new config model given a context. + * + * @param modelContext The model context. + */ + public Model(ConfigModelContext modelContext) { + super(modelContext); + this.context = modelContext; + } + + public ConfigModelContext getContext() { + return context; + } + } + private static class ModelBuilder extends LegacyConfigModelBuilder<Model> { + + public ModelBuilder() { + super(Model.class); + } + + @Override + public void doBuild(Model model, Element element, ConfigModelContext modelContext) { + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromName("foo")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java new file mode 100755 index 00000000000..38925eade28 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.application.Xml; +import com.yahoo.text.XML; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.config.GenericConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + */ +public class VespaDomBuilderTest { + + final static String hosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts>" + + " <host name=\"localhost\">" + + " <alias>node1</alias>" + + " <alias>node2</alias>" + + " </host>" + + "</hosts>"; + + final static String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <config name=\"standard\">" + + " <basicStruct>" + + " <stringVal>default</stringVal>" + + " </basicStruct>" + + " </config> " + + " <config name=\"container.core.container-http\"><port><search>6745</search></port></config>" + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + " <container version=\"1.0\">" + + " <config name=\"standard\">" + + " <basicStruct>" + + " <stringVal>qrservers</stringVal>" + + " </basicStruct>" + + " </config> " + + " <nodes>\n" + + " <node hostalias=\"node1\"/>\n" + + " </nodes>\n" + + " </container>\n" + + "</services>"; + + final static String servicesWithNamespace = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <config name=\"testnamespace\" namespace=\"foo\">" + + " <basicStruct>" + + " <stringVal>default</stringVal>" + + " </basicStruct>" + + " </config> " + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + "</services>"; + + final static String servicesWithNamespace2 = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services>" + + " <config name=\"foo.testnamespace\">" + + " <basicStruct>" + + " <stringVal>default</stringVal>" + + " </basicStruct>" + + " </config> " + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + "</services>"; + + + @Test + public void testUserConfigsWithNamespace() throws Exception { + VespaModel model = createModel(hosts, servicesWithNamespace); + + GenericConfig.GenericConfigBuilder builder = + new GenericConfig.GenericConfigBuilder(new ConfigDefinitionKey("testnamespace", "foo"), new ConfigPayloadBuilder()); + model.getConfig(builder, "admin"); + assertEquals(builder.getPayload().toString(), "{\n" + + " \"basicStruct\": {\n" + + " \"stringVal\": \"default\"\n" + + " }\n" + + "}\n"); + + model = createModel(hosts, servicesWithNamespace2); + + builder = new GenericConfig.GenericConfigBuilder(new ConfigDefinitionKey("testnamespace", "foo"), new ConfigPayloadBuilder()); + model.getConfig(builder, "admin"); + assertEquals(builder.getPayload().toString(), "{\n" + + " \"basicStruct\": {\n" + + " \"stringVal\": \"default\"\n" + + " }\n" + + "}\n"); + } + + @Test + public void testGetElement() { + Element e = Xml.getElement(new StringReader("<searchchain><foo>sdf</foo></searchchain>")); + assertEquals(e.getTagName(), "searchchain"); + assertEquals(XML.getChild(e, "foo").getTagName(), "foo"); + assertEquals(XML.getValue(XML.getChild(e, "foo")), "sdf"); + } + + @Test + public void testHostSystem() throws IOException, SAXException { + VespaModel model = createModel(hosts, services); + HostSystem hostSystem = model.getHostSystem(); + System.out.println(hostSystem); + assertThat(hostSystem.getHosts().size(), is(1)); + HostResource host = hostSystem.getHosts().get(0); + assertThat(host, is(hostSystem.getHostByHostname(host.getHostName()))); + assertNotNull(hostSystem.getHost("node1")); + assertThat(hostSystem.toString(), is("host '" + host.getHostName() + "'")); + } + + private VespaModel createModel(String hosts, String services) { + VespaModelCreatorWithMockPkg creator = new VespaModelCreatorWithMockPkg(hosts, services); + return creator.create(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilderTest.java new file mode 100644 index 00000000000..482e1070a9e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilderTest.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * Basic tests of DependencyBuilder + * @author tonytv + */ +public class DependenciesBuilderTest extends DomBuilderTest { + private Set<String> set(String str) { + Set<String> symbols = new HashSet<>(); + for (String symbol : str.split(",")) { + symbols.add(symbol); + } + return symbols; + } + + @Test + public void testBuildDependencies() { + DependenciesBuilder dependenciesBuilder = new DependenciesBuilder(parse( + "<searcher provides='symbol1 symbol2 ' before='p1' after=' s1' >", + " <provides> symbol3 </provides>", + " <provides> symbol4 </provides>", + " <before> p2 </before>", + " <after>s2</after>", + "</searcher>")); + + Dependencies dependencies = dependenciesBuilder.build(); + + assertEquals(dependencies.provides(), + set("symbol1,symbol2,symbol3,symbol4")); + + assertEquals(dependencies.before(), + set("p1,p2")); + + assertEquals(dependencies.after(), + set("s1,s2")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java new file mode 100644 index 00000000000..32f3453c2ee --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilderTest.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.vespa.model.container.search.searchchain.FederationSearcher; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNotNull; +import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Test of DomFederationSearcherBuilder. + * @author tonytv + */ +public class DomFederationSearcherBuilderTest extends DomBuilderTest { + + @Test + public void ensureCorrectModel() { + FederationSearcher searcher = new DomFederationSearcherBuilder().doBuild(root, parse( + "<federation id='theId'>", + " <provides>p2</provides>", + " <source-set inherits=\"default\" />", + " <source id='source1'>", + " <federationoptions optional='true' />", + " </source>", + " <source id='source2' />", + "</federation>")); + + FederationSearcherModel model = searcher.model; + + assertEquals("theId", model.bundleInstantiationSpec.id.stringValue()); + assertEquals(com.yahoo.search.federation.FederationSearcher.class.getName(), + model.bundleInstantiationSpec.classId.stringValue()); + + assertEquals(2, model.targets.size()); + assertTrue("source-set option was ignored", model.inheritDefaultSources); + + assertThat(targetNames(model.targets), + hasItems("source1", "source2")); + + } + + private List<String> targetNames(List<FederationSearcherModel.TargetSpec> targets) { + List<String> res = new ArrayList<>(); + for (FederationSearcherModel.TargetSpec target : targets) { + res.add(target.sourceSpec.getName()); + } + return res; + } + + @Test + public void require_that_target_selector_can_be_configured() { + FederationSearcher searcher = new DomFederationSearcherBuilder().doBuild(root, parse( + "<federation id='federation-id'>", + " <target-selector id='my-id' class='my-class' />", + "</federation>")); + + String targetSelectorId = "my-id@federation-id"; + + AbstractConfigProducer<?> targetSelector = searcher.getChildren().get(targetSelectorId); + assertNotNull("No target selector child found", targetSelector); + + FederationConfig.Builder builder = new FederationConfig.Builder(); + searcher.getConfig(builder); + assertThat(new FederationConfig(builder).targetSelector(), is(targetSelectorId)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java new file mode 100755 index 00000000000..ef3e84a300a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; +import com.yahoo.vespa.model.container.search.searchchain.HttpProviderSearcher; +import com.yahoo.vespa.model.container.search.searchchain.Provider; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.HashMap; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public class DomProviderBuilderTest extends DomBuilderTest { + + private static final Element noProxy = parse( + "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", + " <nodes>", + " <node host='sourcehost' port='12'/>", + " </nodes>", + "</provider>"); + + private static final Element defaultProxy = parse( + "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", + " <yca-proxy/>", + " <nodes>", + " <node host='sourcehost' port='12'/>", + " </nodes>", + "</provider>"); + + private static final Element proprietaryProxy = parse( + "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", + " <yca-proxy host='my-host' port='80'/>", + " <nodes>", + " <node host='sourcehost' port='12'/>", + " </nodes>", + "</provider>"); + + private static final Element illegal_proxyWithoutId= parse( + "<provider id='yca-provider' type='vespa'>", + " <yca-proxy host='my-host' port='80'/>", + " <nodes>", + " <node host='sourcehost' port='12'/>", + " </nodes>", + "</provider>"); + + private Provider provider; + + @Test + public void testYcaConfig_noProxy() { + provider = new DomProviderBuilder(new HashMap<String, ComponentsBuilder.ComponentType>()).doBuild(root, noProxy); + + ChainedComponent providerSearcher = provider.getInnerComponents().iterator().next(); + assertThat(providerSearcher, instanceOf(HttpProviderSearcher.class)); + + ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); + ((HttpProvider)provider).getConfig(providerBuilder); + ProviderConfig providerConfig = new ProviderConfig(providerBuilder); + assertThat(providerConfig.yca().applicationId(), is("my-app")); + assertThat(providerConfig.yca().useProxy(), is(false)); + } + + @Test + public void testYcaConfig_defaultProxy() { + provider = new DomProviderBuilder(new HashMap<String, ComponentsBuilder.ComponentType>()).doBuild(root, defaultProxy); + + ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); + ((HttpProvider)provider).getConfig(providerBuilder); + ProviderConfig providerConfig = new ProviderConfig(providerBuilder); + + assertThat(providerConfig.yca().applicationId(), is("my-app")); + assertThat(providerConfig.yca().useProxy(), is(true)); + assertThat(providerConfig.yca().host(), is("yca-proxy.corp.yahoo.com")); // default from def-file + assertThat(providerConfig.yca().port(), is(3128)); // default from def-file + } + + @Test + public void testYcaConfig_proprietaryProxy() { + provider = new DomProviderBuilder(new HashMap<String, ComponentsBuilder.ComponentType>()).doBuild(root, proprietaryProxy); + + ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); + ((HttpProvider)provider).getConfig(providerBuilder); + ProviderConfig providerConfig = new ProviderConfig(providerBuilder); + + assertThat(providerConfig.yca().applicationId(), is("my-app")); + assertThat(providerConfig.yca().useProxy(), is(true)); + assertThat(providerConfig.yca().host(), is("my-host")); + assertThat(providerConfig.yca().port(), is(80)); + } + + @Test + public void testFail_ycaProxyWithoutId() { + try { + provider = new DomProviderBuilder(new HashMap<String, ComponentsBuilder.ComponentType>()).doBuild(root, illegal_proxyWithoutId); + fail("Expected exception upon illegal xml."); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Provider 'yca-provider' must have a YCA application ID, since a YCA proxy is given")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java new file mode 100644 index 00000000000..98469bf26c1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.search.searchchain.*; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static com.yahoo.container.core.ChainsConfig.Chains; +import static com.yahoo.container.core.ChainsConfig.Components; +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.hamcrest.Matchers.containsString; + +/** + * Test of Search chains builder. + * @author tonytv + */ +public class DomSearchChainsBuilderTest extends DomBuilderTest { + private SearchChains searchChains; + + private static final Element element = parse( + "<searchchains>", + " <searcher id='searcher:1'/>", + + " <provider id='provider:1' type='vespa' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", + " cacheweight='2.3'>", + " <federationoptions optional='true' timeout='2.3 s' />", + " <nodes>", + " <node host='sourcehost' port='12'/>", + " </nodes>", + + " <source id='source:1' inherits='parentChain3 parentChain4' excludes='ExcludedSearcher3 ExcludedSearcher4'>", + " <federationoptions timeout='12 ms' />", + " </source>", + + " </provider>", + + " <searchchain id='default'>", + " <federation id='federationSearcher'>", + " <source id='mysource'>", + " <federationoptions optional='false' />", + " </source>", + " </federation>", + " </searchchain>", + + "</searchchains>"); + + + @Before + public void createSearchChains() { + searchChains = new DomSearchChainsBuilder().build(root, element); + } + + @Test + public void referToFederationAsSearcher() { + final Element element = parse( + "<searchchains>", + " <federation id='federationSearcher'>", + " <source id='mysource'>", + " <federationoptions optional='false' />", + " </source>", + " </federation>", + + " <searchchain id='default'>", + " <searcher id='federationSearcher'/>", + " </searchchain>", + "</searchchains>"); + + try { + new DomSearchChainsBuilder().build(new MockRoot(), element); + fail("Expected exception when referring to an outer 'federation' as a 'searcher'."); + } catch (RuntimeException e) { + assertThat(e.getMessage(), containsString("Two different types declared for the component with name 'federationSearcher'")); + } + } + + @Test + public void ensureSearchChainsExists() { + for (String id : Arrays.asList("provider:1", "source:1@provider:1", "default")) { + assertNotNull("Missing search chain " + id, getSearchChain(id)); + } + } + + @Test + public void ensureSearcherExists() { + assertThat(searchChains.allComponents(), hasItem(searcherWithId("searcher:1"))); + } + + private Matcher<ChainedComponent<?>> searcherWithId(final String componentId) { + return new BaseMatcher<ChainedComponent<?>>() { + @Override + public boolean matches(Object o) { + return o instanceof ChainedComponent && + ((ChainedComponent) o).getComponentId().equals(new ComponentId(componentId)); + } + + @Override + public void describeTo(Description description) { + description.appendText("a searcher with id ").appendValue(componentId); + } + }; + } + + @Test + public void checkProviderFederationOptions() { + FederationOptions options = getProvider().federationOptions(); + + assertEquals(true, options.getOptional()); + assertEquals(2300, options.getTimeoutInMilliseconds()); + } + + @Test + public void checkSourceFederationOptions() { + FederationOptions options = getSource().federationOptions(); + + assertEquals(true, options.getOptional()); //inherited + assertEquals(12, options.getTimeoutInMilliseconds()); + } + + @Test + public void checkDefaultTargets() { + Collection<? extends GenericTarget> defaultTargets = + getProvider().defaultFederationTargets(); + + assertEquals(1, defaultTargets.size()); + assertEquals(getSearchChain("source:1@provider:1"), first(defaultTargets)); + } + + @Test + public void checkInnerSearcherIdIsNestedInSearchChainId() { + ChainsConfig.Builder builder = new ChainsConfig.Builder(); + searchChains.getConfig(builder); + ChainsConfig config = new ChainsConfig(builder); + + checkInnerSearcherIdIsNestedInSearchChainId(config, "federationSearcher", "default"); + checkInnerSearcherIdIsNestedInSearchChainId(config, "VespaSearcher", "provider"); + } + + private void checkInnerSearcherIdIsNestedInSearchChainId(ChainsConfig config, + String partOfSearcherName, + String searchChainName) { + Components searcher = getSearcherConfig(config.components(), partOfSearcherName); + ComponentId searcherId = ComponentId.fromString(searcher.id()); + + assertThat(searcherId.getNamespace(), is(getSearchChain(searchChainName).getComponentId())); + + Chains searchChain = getSearchChainConfig(config.chains(), searchChainName); + assertThat(ComponentId.fromString(searchChain.components(0)), is(searcherId)); + } + + private Chains getSearchChainConfig(List<Chains> searchChains, + String searchChainName) { + for (Chains searchChain : searchChains) { + if (ComponentId.fromString(searchChain.id()).getName().equals(searchChainName)) + return searchChain; + } + fail("No search chain matching " + searchChainName); + return null; + } + + private Components getSearcherConfig(List<Components> searchers, String partOfId) { + for (Components searcher : searchers) { + if (searcher.id().contains(partOfId)) + return searcher; + } + fail("No searcher matching " + partOfId); + return null; + } + + private static <T> T first(Iterable<T> coll) { + return coll.iterator().next(); + } + + private Provider getProvider() { + return (Provider)getSearchChain("provider:1"); + } + + private Source getSource() { + return first(getProvider().getSources()); + } + + private SearchChain getSearchChain(String componentSpecification) { + return searchChains.allChains().getComponent(new ComponentSpecification(componentSpecification)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilderTest.java new file mode 100644 index 00000000000..b77161ef415 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilderTest.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import org.junit.Test; + +import static junit.framework.TestCase.assertEquals; + + +/** + * @author tonytv + */ +public class DomSearcherBuilderTest extends DomBuilderTest { + @Test + public void ensureCorrectModel() { + ChainedComponent<ChainedComponentModel> searcher = new DomSearcherBuilder().doBuild(root, parse( + "<searcher id='theId' class='theclassid' bundle='thebundle' provides='p1'>", + " <provides>p2</provides>", + "</searcher>")); + + ChainedComponentModel model = searcher.model; + assertEquals(2, model.dependencies.provides().size()); + + BundleInstantiationSpecification instantiationSpecification = model.bundleInstantiationSpec; + assertEquals("theId", instantiationSpecification.id.stringValue()); + assertEquals("theclassid", instantiationSpecification.classId.stringValue()); + assertEquals("thebundle", instantiationSpecification.bundle.stringValue()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/searchchains/.gitignore b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/searchchains/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/searchchains/.gitignore diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java new file mode 100644 index 00000000000..5987ab8410a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/Gateway20TestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients.test; + +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.QrConfig; +import com.yahoo.container.QrConfig.Builder; +import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class Gateway20TestCase { + private static String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model + + @Test + public void testSimpleDocprocV3() throws Exception { + VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/simpleconfig.v2.docprocv3").create(); + QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); + assertEquals(qrConfig.rpc().enabled(), true); + assertEquals("filedistribution/" + hostname, qrConfig.filedistributor().configid()); + assertEquals("container.container.0", qrConfig.discriminator()); + + ContainerHttpConfig cHConfig = new ContainerHttpConfig((ContainerHttpConfig.Builder) model.getConfig(new ContainerHttpConfig.Builder(), "container/container.0")); + assertTrue(cHConfig.enabled()); + assertEquals(Defaults.getDefaults().vespaWebServicePort(), cHConfig.port().search()); + + ComponentsConfig componentsConfig = new ComponentsConfig((ComponentsConfig.Builder) model.getConfig(new ComponentsConfig.Builder(), "container/container.0")); + ArrayList<String> components = new ArrayList<>(); + for (ComponentsConfig.Components component : componentsConfig.components()) { + components.add(component.id()); + } + List<String> expectedComponents = Arrays.asList("com.yahoo.docproc.jdisc.DocumentProcessingHandler", + "com.yahoo.feedhandler.VespaFeedHandler", + "com.yahoo.feedhandler.VespaFeedHandlerCompatibility", + "com.yahoo.feedhandler.VespaFeedHandlerGet", + "com.yahoo.feedhandler.VespaFeedHandlerRemove", + "com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", + "com.yahoo.feedhandler.VespaFeedHandlerStatus", + "com.yahoo.feedhandler.VespaFeedHandlerVisit", + "com.yahoo.search.handler.SearchHandler", + "com.yahoo.container.jdisc.state.StateHandler"); + assertTrue(components.containsAll(expectedComponents)); + } + + @Test + public void testAdvanced() throws Exception { + VespaModel model = new VespaModelCreatorWithFilePkg("src/test/cfg/clients/advancedconfig.v2").create(); + + QrConfig qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); + assertEquals(qrConfig.rpc().enabled(), true); + assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname); + assertEquals("container.container.0", qrConfig.discriminator()); + + ContainerHttpConfig cHConfig = new ContainerHttpConfig((ContainerHttpConfig.Builder) model.getConfig(new ContainerHttpConfig.Builder(), "container/container.0")); + assertTrue(cHConfig.enabled()); + assertEquals(Defaults.getDefaults().vespaWebServicePort(), cHConfig.port().search()); + + qrConfig = new QrConfig((Builder) model.getConfig(new QrConfig.Builder(), "container/container.0")); + assertEquals(qrConfig.rpc().enabled(), true); + assertEquals(qrConfig.filedistributor().configid(), "filedistribution/" + hostname); + + cHConfig = new ContainerHttpConfig((ContainerHttpConfig.Builder) model.getConfig(new ContainerHttpConfig.Builder(), "container/container.0")); + assertEquals(cHConfig.enabled(), true); + assertEquals(Defaults.getDefaults().vespaWebServicePort(), cHConfig.port().search()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java new file mode 100644 index 00000000000..9a6c57da141 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients.test; + +import com.yahoo.vespa.config.content.spooler.SpoolerConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.CommonVespaModelSetup; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.util.*; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class SpoolerTestCase extends junit.framework.TestCase { + + public void testSimple() throws Exception { + VespaModel model = createModel("src/test/cfg/clients/simpleconfig.v2.docprocv3"); + + SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); + SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); + SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); + LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); + parameters.put("route", "default"); + parameters.put("foo", "bar"); + SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", + parameters); + builder.maxfailuresize(100000). + maxfatalfailuresize(1000000). + threads(5). + parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); + final int spoolerIndex = 0; + testSpoolerConfigBuilder(model, spoolerIndex, builder); + + FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). + abortondocumenterror(false). + maxpendingbytes(8000). + tracelevel(7); + testFeederConfigBuilder(model, spoolerIndex, feederBuilder); + } + + public void testAdvanced() throws Exception { + VespaModel model = createModel("src/test/cfg/clients/advancedconfig.v2"); + + SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); + SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); + SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); + LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); + parameters.put("route", "default"); + SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", + parameters); + builder.keepsuccess(true). + parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); + int spoolerIndex = 0; + testSpoolerConfigBuilder(model, spoolerIndex, builder); + + FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). + abortondocumenterror(false). + maxpendingbytes(8000). + timeout(90.0); + testFeederConfigBuilder(model, spoolerIndex, feederBuilder); + + builder = new SpoolerConfig.Builder(); + parameters = new LinkedHashMap<>(); + parameters.put("route", "othercluster"); + + parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", + parameters); + builder.keepsuccess(false). + parsers(parserBuilder1); + spoolerIndex = 1; + testSpoolerConfigBuilder(model, spoolerIndex, builder); + + feederBuilder = new FeederConfig.Builder(). + abortondocumenterror(false). + maxpendingbytes(4000). + timeout(50.0); + testFeederConfigBuilder(model, spoolerIndex, feederBuilder); + + builder = new SpoolerConfig.Builder(); + parserBuilder1 = new SpoolerConfig.Parsers.Builder(); + parserBuilder1.classname("com.yahoo.vespaspooler.MusicFileParser"); + builder.parsers(parserBuilder1); + String id = "plan9"; + testSpoolerConfigBuilder(model, "clients/spoolers/" + id, builder); + + feederBuilder = new FeederConfig.Builder(). + route("myroute"). + mbusport(14064). + timeout(90.0); + testFeederConfigBuilder(model, "clients/spoolers/" + id, feederBuilder); + } + + SpoolerConfig.Parsers.Builder createParserBuilder(String className) { + return createParserBuilder(className, new HashMap<String, String>()); + } + + SpoolerConfig.Parsers.Builder createParserBuilder(String className, Map<String, String> parameters) { + SpoolerConfig.Parsers.Builder builder = new SpoolerConfig.Parsers.Builder(); + builder.classname(className); + if (!parameters.isEmpty()) { + List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); + for (Map.Entry<String, String> entry : parameters.entrySet()) { + final SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); + parametersBuilder.key(entry.getKey()).value(entry.getValue()); + parametersBuilders.add(parametersBuilder); + } + builder.parameters(parametersBuilders); + } + return builder; + } + + private void testSpoolerConfigBuilder(VespaModel model, int index, SpoolerConfig.Builder expected) throws Exception { + testSpoolerConfigBuilder(model, "clients/spoolers/spooler." + index, expected); + } + + private void testSpoolerConfigBuilder(VespaModel model, String id, SpoolerConfig.Builder expected) throws Exception { + SpoolerConfig.Builder b = new SpoolerConfig.Builder(); + model.getConfig(b, id); + SpoolerConfig config = new SpoolerConfig(b); + final SpoolerConfig expectedConfig = new SpoolerConfig(expected); + assertEquals(expectedConfig, config); + } + + private void testFeederConfigBuilder(VespaModel model, int index, FeederConfig.Builder expected) throws Exception { + testFeederConfigBuilder(model, "clients/spoolers/spooler." + index, expected); + } + + private void testFeederConfigBuilder(VespaModel model, String id, FeederConfig.Builder expected) throws Exception { + FeederConfig.Builder b = new FeederConfig.Builder(); + model.getConfig(b, id); + FeederConfig config = new FeederConfig(b); + final FeederConfig expectedConfig = new FeederConfig(expected); + assertEquals(expectedConfig, config); + } + + private VespaModel createModel(String configFile) throws Exception { + return CommonVespaModelSetup.createVespaModelWithMusic(configFile); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java new file mode 100755 index 00000000000..3365177409a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.RoutingProviderConfig; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ContainerClusterTest { + + @Test + public void requireThatDefaultMetricConsumerFactoryCanBeConfigured() { + ContainerCluster cluster = newContainerCluster(); + cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); + assertEquals(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD, + getMetricDefaultsConfig(cluster).factory()); + } + + @Test + public void requireThatDefaultMetricConsumerFactoryMatchesConfigDefault() { + ContainerCluster cluster = newContainerCluster(); + assertEquals(new MetricDefaultsConfig(new MetricDefaultsConfig.Builder()).factory(), + getMetricDefaultsConfig(cluster).factory()); + } + + @Test + public void requireThatClusterInfoIsPopulated() { + ContainerCluster cluster = newContainerCluster(); + ClusterInfoConfig config = getClusterInfoConfig(cluster); + assertEquals("name", config.clusterId()); + assertEquals(2, config.nodeCount()); + assertEquals(2, config.services().size()); + + Iterator<ClusterInfoConfig.Services> iterator = config.services().iterator(); + ClusterInfoConfig.Services service = iterator.next(); + assertEquals("host-c1", service.hostname()); + assertEquals(0, service.index()); + assertEquals(4, service.ports().size()); + + service = iterator.next(); + assertEquals("host-c2", service.hostname()); + assertEquals(1, service.index()); + assertEquals(4, service.ports().size()); + } + + @Test + public void requreThatWeCanGetTheZoneConfig() { + DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(true).build()) + .zone(new Zone(Environment.test, RegionName.from("some-region"))).build(); + MockRoot root = new MockRoot("foo", state); + ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); + cluster.getConfig(builder); + ConfigserverConfig config = new ConfigserverConfig(builder); + assertEquals(Environment.test.value(), config.environment()); + assertEquals("some-region", config.region()); + } + + private ContainerCluster createContainerCluster(boolean isHosted) { + DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(); + MockRoot root = new MockRoot("foo", state); + ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); + cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"), new ContainerSearch.Options())); + return cluster; + } + private void verifyHeapSizeAsPercentageOfPhysicalMemory(boolean isHosted, int percentage) { + ContainerCluster cluster = createContainerCluster(isHosted); + + QrStartConfig.Builder qsB = new QrStartConfig.Builder(); + cluster.getSearch().getConfig(qsB); + QrStartConfig qsC= new QrStartConfig(qsB); + assertEquals(percentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory()); + } + + @Test + public void requireThatHeapSizeAsPercentageOfPhysicalMemoryForHostedAndNot() { + verifyHeapSizeAsPercentageOfPhysicalMemory(true, 33); + verifyHeapSizeAsPercentageOfPhysicalMemory(false, 0); + } + + private void verifyJvmArgs(boolean isHosted, boolean hasDocproc, String expectedArgs, String jvmArgs) { + if (isHosted && hasDocproc) { + String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; + if ( ! "".equals(expectedArgs)) { + defaultHostedJVMArgs = defaultHostedJVMArgs + " "; + } + assertEquals(defaultHostedJVMArgs + expectedArgs, jvmArgs); + } else { + assertEquals(expectedArgs, jvmArgs); + } + } + private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) { + ContainerCluster cluster = createContainerCluster(isHosted); + if (hasDocProc) { + cluster.setDocproc(new ContainerDocproc(cluster, null)); + } + addContainer(cluster, "c1", "host-c1"); + assertEquals(1, cluster.getContainers().size()); + Container container = cluster.getContainers().get(0); + verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmArgs()); + container.setJvmArgs("initial"); + verifyJvmArgs(isHosted, hasDocProc, "initial", container.getJvmArgs()); + container.prependJvmArgs("ignored"); + verifyJvmArgs(isHosted, hasDocProc, "ignored initial", container.getJvmArgs()); + container.appendJvmArgs("override"); + verifyJvmArgs(isHosted, hasDocProc, "ignored initial override", container.getJvmArgs()); + container.setJvmArgs(null); + verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmArgs()); + } + @Test + public void requireThatJvmArgsControlWorksForHostedAndNot() { + verifyJvmArgs(true, false); + verifyJvmArgs(true, true); + verifyJvmArgs(false, false); + verifyJvmArgs(false, true); + } + + private void verifyThatWeCanHandleNull(boolean isHosted) { + + } + @Test + public void requireThatWeCanhandleNull() { + ContainerCluster cluster = createContainerCluster(false); + addContainer(cluster, "c1", "host-c1"); + Container container = cluster.getContainers().get(0); + container.setJvmArgs(""); + String empty = container.getJvmArgs(); + container.setJvmArgs(null); + assertEquals(empty, container.getJvmArgs()); + } + + @Test + public void requireThatRoutingProviderIsDisabledForNonHosted() { + DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(false).build()).build(); + MockRoot root = new MockRoot("foo", state); + ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); + RoutingProviderConfig.Builder builder = new RoutingProviderConfig.Builder(); + cluster.getConfig(builder); + RoutingProviderConfig config = new RoutingProviderConfig(builder); + assertFalse(config.enabled()); + assertEquals(0, cluster.getAllComponents().stream().map(c -> c.getClassId().getName()).filter(c -> c.equals("com.yahoo.jdisc.http.filter.security.RoutingConfigProvider")).count()); + } + + private static void addContainer(ContainerCluster cluster, String name, String hostName) { + Container container = new Container(cluster, name); + container.setHostResource(new HostResource(new Host(null, hostName))); + container.initService(); + cluster.addContainer(container); + } + + private static ContainerCluster newContainerCluster() { + ContainerCluster cluster = new ContainerCluster(null, "subId", "name"); + addContainer(cluster, "c1", "host-c1"); + addContainer(cluster, "c2", "host-c2"); + return cluster; + } + + private static MetricDefaultsConfig getMetricDefaultsConfig(ContainerCluster cluster) { + MetricDefaultsConfig.Builder builder = new MetricDefaultsConfig.Builder(); + cluster.getConfig(builder); + return new MetricDefaultsConfig(builder); + } + + private static ClusterInfoConfig getClusterInfoConfig(ContainerCluster cluster) { + ClusterInfoConfig.Builder builder = new ClusterInfoConfig.Builder(); + cluster.getConfig(builder); + return new ClusterInfoConfig(builder); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java new file mode 100644 index 00000000000..7173e05a4c1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.processing.ProcessingChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.13 + */ +public class ContainerIncludeTest { + + @Test + public void include() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude/"); + VespaModel model = creator.create(); + + assertThat(model.getContainerClusters().size(), is(1)); + ContainerCluster cluster = model.getContainerClusters().values().iterator().next(); + + assertThat(cluster.getSearchChains(), notNullValue()); + + Map<String, SearchChain> searchChainMap = new HashMap<>(); + for (SearchChain searchChain : cluster.getSearchChains().allChains().allComponents()) { + searchChainMap.put(searchChain.getId().stringValue(), searchChain); + } + assertThat(searchChainMap.get("searchchain1"), notNullValue()); + assertThat(searchChainMap.get("searchchain1").getInnerComponents().size(), is(1)); + assertThat(searchChainMap.get("searchchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher1")); + + assertThat(searchChainMap.get("searchchain2"), notNullValue()); + assertThat(searchChainMap.get("searchchain2").getInnerComponents().size(), is(1)); + assertThat(searchChainMap.get("searchchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher2")); + + assertThat(searchChainMap.get("searchchain3"), notNullValue()); + assertThat(searchChainMap.get("searchchain3").getInnerComponents().size(), is(1)); + assertThat(searchChainMap.get("searchchain3").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher3")); + + assertThat(searchChainMap.get("searchchain4"), notNullValue()); + assertThat(searchChainMap.get("searchchain4").getInnerComponents().size(), is(1)); + assertThat(searchChainMap.get("searchchain4").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Searcher4")); + + + assertThat(cluster.getDocprocChains(), notNullValue()); + + Map<String, DocprocChain> docprocChainMap = new HashMap<>(); + for (DocprocChain docprocChain : cluster.getDocprocChains().allChains().allComponents()) { + docprocChainMap.put(docprocChain.getId().stringValue(), docprocChain); + } + + assertThat(docprocChainMap.get("docprocchain1"), notNullValue()); + assertThat(docprocChainMap.get("docprocchain1").getInnerComponents().size(), is(1)); + assertThat(docprocChainMap.get("docprocchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.DocumentProcessor1")); + + assertThat(docprocChainMap.get("docprocchain2"), notNullValue()); + assertThat(docprocChainMap.get("docprocchain2").getInnerComponents().size(), is(1)); + assertThat(docprocChainMap.get("docprocchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.DocumentProcessor2")); + + + assertThat(cluster.getProcessingChains(), notNullValue()); + + Map<String, ProcessingChain> processingChainMap = new HashMap<>(); + for (ProcessingChain processingChain : cluster.getProcessingChains().allChains().allComponents()) { + processingChainMap.put(processingChain.getId().stringValue(), processingChain); + } + + assertThat(processingChainMap.get("processingchain1"), notNullValue()); + assertThat(processingChainMap.get("processingchain1").getInnerComponents().size(), is(1)); + assertThat(processingChainMap.get("processingchain1").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Processor1")); + + assertThat(processingChainMap.get("processingchain2"), notNullValue()); + assertThat(processingChainMap.get("processingchain2").getInnerComponents().size(), is(1)); + assertThat(processingChainMap.get("processingchain2").getInnerComponents().iterator().next().getComponentId().stringValue(), is("com.yahoo.Processor2")); + } + + @Test(expected = IllegalArgumentException.class) + public void includeNonExistent() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude2/"); + creator.create(); + } + + @Test(expected = IllegalArgumentException.class) + public void includeAbsolutePath() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude3/"); + creator.create(); + } + + @Test(expected = IllegalArgumentException.class) + public void includeNonDirectory() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude4/"); + creator.create(); + } + + @Test(expected = IllegalArgumentException.class) + public void include_file_with_wrong_root_element_name() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude5/"); + creator.create(); + } + + @Test + public void include_empty_directory() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/containerinclude6/"); + creator.create(); + } + + @Test + public void included_file_with_xml_schema_violation() throws Exception { + try { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/container/data/include_xml_error/"); + creator.create(true); + fail("Expected exception due to xml schema violation ('zearcer')"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("XML error")); + assertThat(e.getMessage(), containsString("zearcer")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java new file mode 100644 index 00000000000..170586bef85 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.configserver; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.jdisc.metrics.yamasconsumer.cloud.ScoreBoardConfig; +import com.yahoo.net.HostName; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.17 + */ +public class ConfigserverClusterTest { + + private AbstractConfigProducerRoot root; + + @Before + public void setupCluster() { + String services = "<jdisc id='standalone' version='1.0'>" + + " <http>" + + " <server port='1337' id='configserver' />" + + " </http>" + + "</jdisc>"; + root = new MockRoot(); + new ConfigServerContainerModelBuilder(new TestOptions().rpcPort(12345).useVespaVersionInRequest(true) + .hostedVespa(true).environment("test").region("bar") + .numParallelTenantLoaders(4)) + .build(new DeployState.Builder().build(), null, root, XML.getDocument(services).getDocumentElement()); + root.freezeModelTopology(); + } + + @Test + public void testStatisticsConfig() { + StatisticsConfig config = root.getConfig(StatisticsConfig.class, "configserver/standalone"); + assertThat((int) config.collectionintervalsec(), is(60)); + assertThat((int) config.loggingintervalsec(), is(60)); + } + + @Test + public void testScoreBoardConfig() { + ScoreBoardConfig config = root.getConfig(ScoreBoardConfig.class, "configserver/standalone"); + assertThat(config.applicationName(), is("configserver")); + assertThat(config.flushTime(), is(60)); + assertThat(config.step(), is(60)); + } + + @Test + public void testHealthMonitorConfig() { + HealthMonitorConfig config = root.getConfig(HealthMonitorConfig.class, "configserver/standalone"); + assertThat(((int) config.snapshot_interval()), is(60)); + } + + @Test + public void testConfigserverConfig() { + ConfigserverConfig config = root.getConfig(ConfigserverConfig.class, "configserver/standalone"); + assertThat(config.configModelPluginDir().size(), is(1)); + assertThat(config.configModelPluginDir().get(0), is(Defaults.getDefaults().vespaHome() + "lib/jars/config-models")); + assertThat(config.rpcport(), is(12345)); + assertThat(config.httpport(), is(1337)); + assertThat(config.serverId(), is(HostName.getLocalhost())); + assertTrue(config.useVespaVersionInRequest()); + assertThat(config.numParallelTenantLoaders(), is(4)); + assertFalse(config.multitenant()); + assertTrue(config.hostedVespa()); + assertThat(config.environment(), is("test")); + assertThat(config.region(), is("bar")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java new file mode 100644 index 00000000000..b423a2b2305 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.configserver; + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; + +import java.util.Optional; + +/** + * @author lulf + * @since 5. + */ +public class TestOptions implements CloudConfigOptions { + private Optional<Integer> rpcPort = Optional.empty(); + private Optional<String> environment = Optional.empty(); + private Optional<String> region = Optional.empty(); + private Optional<String> defaultFlavor = Optional.empty(); + private Optional<String> defaultAdminFlavor = Optional.empty(); + private Optional<String> defaultContainerFlavor = Optional.empty(); + private Optional<String> defaultContentFlavor = Optional.empty(); + private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); + private Optional<Boolean> hostedVespa = Optional.empty(); + private Optional<Integer> numParallelTenantLoaders = Optional.empty(); + + @Override + public Optional<Integer> rpcPort() { + return rpcPort; + } + + public TestOptions rpcPort(int port) { + this.rpcPort = Optional.of(port); + return this; + } + + public TestOptions useVespaVersionInRequest(boolean useVespaVersionInRequest) { + this.useVespaVersionInRequest = Optional.of(useVespaVersionInRequest); + return this; + } + + @Override + public Optional<Boolean> multiTenant() { return Optional.empty(); } + + @Override + public Optional<Boolean> hostedVespa() { + return hostedVespa; + } + + @Override + public ConfigServer[] allConfigServers() { + return new ConfigServer[0]; + } + + @Override + public Optional<Integer> zookeeperClientPort() { + return Optional.empty(); + } + + @Override + public String[] configModelPluginDirs() { + return new String[0]; + } + + @Override + public Optional<Long> sessionLifeTimeSecs() { + return Optional.empty(); + } + + @Override + public Optional<Long> zookeeperBarrierTimeout() { + return Optional.empty(); + } + + @Override + public Optional<Integer> zookeeperElectionPort() { + return Optional.empty(); + } + + @Override + public Optional<Integer> zookeeperQuorumPort() { + return Optional.empty(); + } + + @Override + public Optional<String> payloadCompressionType() { return Optional.empty(); } + + @Override + public Optional<String> environment() { return environment; } + + @Override + public Optional<String> region() { return region; } + + @Override + public Optional<String> defaultFlavor() { return defaultFlavor; } + + @Override + public Optional<String> defaultAdminFlavor() { return defaultAdminFlavor; } + + @Override + public Optional<String> defaultContainerFlavor() { return defaultContainerFlavor; } + + @Override + public Optional<String> defaultContentFlavor() { return defaultContentFlavor; } + + @Override + public Optional<Boolean> useVespaVersionInRequest() { return useVespaVersionInRequest; } + + @Override + public Optional<Integer> numParallelTenantLoaders() { return numParallelTenantLoaders; } + + public TestOptions numParallelTenantLoaders(int numLoaders) { + this.numParallelTenantLoaders = Optional.of(numLoaders); + return this; + } + + public TestOptions environment(String environment) { + this.environment = Optional.of(environment); + return this; + } + + public TestOptions region(String region) { + this.region = Optional.of(region); + return this; + } + + public TestOptions hostedVespa(boolean hostedVespa) { + this.hostedVespa = Optional.of(hostedVespa); + return this; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java new file mode 100644 index 00000000000..8eca10fdc9e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/docproc/StandaloneDocprocContainerTest.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.14 + */ +public class StandaloneDocprocContainerTest extends DomBuilderTest { + + public ContainerCluster setupCluster(boolean standalone) { + ContainerModelBuilder builder = new ContainerModelBuilder(standalone, Networking.disable); + ContainerModel model = builder.build(DeployState.createTestState(), null, root, servicesXml()); + + if (!standalone) + model.getCluster().getDocproc().getChains().addServersAndClientsForChains(); + + root.freezeModelTopology(); + return model.getCluster(); + } + + private Element servicesXml() { + return parse("" + + "<jdisc version=\"1.0\">\n" + + " <document-processing>\n" + + " <chain id=\"foo\">\n" + + " <documentprocessor id=\"MyDocproc\"/>\n" + + " </chain>\n" + + " </document-processing>\n" + + " <nodes>\n" + + " <node hostalias=\"node01\"/>\n" + + " </nodes>\n" + + "</jdisc>\n"); + } + + @Test + public void requireMbusProvidersWhenNonStandalone() { + ContainerCluster containerCluster = setupCluster(false); + Map<ComponentId, Component<?, ?>> components = containerCluster.getComponentsMap(); + + boolean foundAtLeastOneClient = false; + boolean foundAtLeastOneServer = false; + + for (ComponentId componentId : components.keySet()) { + if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; + if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; + } + assertThat(foundAtLeastOneClient, is(true)); + assertThat(foundAtLeastOneServer, is(true)); + + } + + @Test + public void requireNoMbusProvidersWhenStandalone() { + ContainerCluster containerCluster = setupCluster(true); + Map<ComponentId, Component<?, ?>> components = containerCluster.getComponentsMap(); + + boolean foundAtLeastOneClient = false; + boolean foundAtLeastOneServer = false; + + for (ComponentId componentId : components.keySet()) { + if (componentId.stringValue().contains("MbusClient")) foundAtLeastOneClient = true; + if (componentId.stringValue().contains("MbusServer")) foundAtLeastOneServer = true; + } + assertThat(foundAtLeastOneClient, is(false)); + assertThat(foundAtLeastOneServer, is(false)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java new file mode 100644 index 00000000000..92a628120e9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.jdisc.config.HttpServerConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.http.xml.HttpBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.collections.CollectionUtil.first; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertNotNull; + +/** + * @author gjoranv + * @since 5.1.25 + */ +public class FilterBindingsTest extends DomBuilderTest { + + private static final String MY_CHAIN_BINDING = "http://*/my-chain-binding"; + + private Http buildHttp(Element xml) throws Exception { + Http http = new HttpBuilder().build(root, xml); + root.freezeModelTopology(); + http.validate(); + return http; + } + + + private void buildContainerCluster(Element containerElem) throws SAXException, IOException { + ContainerModel model = new ContainerModelBuilder(true, Networking.enable).build(DeployState.createTestState(), null, root, containerElem); + root.freezeModelTopology(); + } + + @Test + public void request_chain_binding_is_added_to_http() throws Exception { + Element xml = parse( + "<http>", + " <filtering>", + " <request-chain id='my-request-chain'>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", + " </request-chain>", + " </filtering>", + "</http>"); + Http http = buildHttp(xml); + + Http.Binding binding = first(http.bindings); + assertThat(binding.filterId.getName(), is("my-request-chain")); + assertThat(binding.binding, is(MY_CHAIN_BINDING)); + + Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-request-chain"); + assertNotNull("Missing chain", myChain); + } + + @Test + public void response_chain_binding_is_added_to_http() throws Exception { + Element xml = parse( + "<http>", + " <filtering>", + " <response-chain id='my-response-chain'>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", + " </response-chain>", + " </filtering>", + "</http>"); + Http http = buildHttp(xml); + + Http.Binding binding = first(http.bindings); + assertThat(binding.filterId.getName(), is("my-response-chain")); + assertThat(binding.binding, is(MY_CHAIN_BINDING)); + + Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-response-chain"); + assertNotNull("Missing chain", myChain); + } + + @Test + public void bindings_are_added_to_config_for_all_http_servers_with_jetty() throws Exception { + final Element xml = parse( + "<jdisc version='1.0' jetty='true'>", + " <http>", + " <filtering>", + " <request-chain id='my-request-chain'>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", + " </request-chain>", + " </filtering>", + " <server id='server1' port='8000' />", + " <server id='server2' port='9000' />", + " </http>", + "</jdisc>"); + buildContainerCluster(xml); + + { + final ServerConfig config = root.getConfig(ServerConfig.class, "jdisc/http/jdisc-jetty/server1"); + assertThat(config.filter().size(), is(1)); + assertThat(config.filter(0).id(), is("my-request-chain")); + assertThat(config.filter(0).binding(), is(MY_CHAIN_BINDING)); + } + { + final ServerConfig config = root.getConfig(ServerConfig.class, "jdisc/http/jdisc-jetty/server2"); + assertThat(config.filter().size(), is(1)); + assertThat(config.filter(0).id(), is("my-request-chain")); + assertThat(config.filter(0).binding(), is(MY_CHAIN_BINDING)); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java new file mode 100644 index 00000000000..8966f866fb8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterChainsTest.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.http.xml.HttpBuilder; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import static com.yahoo.collections.CollectionUtil.first; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @author tonytv + * @since 5.1.26 + */ +public class FilterChainsTest extends DomBuilderTest { + private Http http; + + @Before + public void setupFilterChains() { + http = new HttpBuilder().build(root, servicesXml()); + root.freezeModelTopology(); + } + + private Element servicesXml() { + return parse( + "<http>", + " <filtering>", + " <filter id='outer' />", + " <request-chain id='myChain'>", + " <filter id='inner' />", + " </request-chain>", + " </filtering>", + "</http>"); + } + + @Test + public void chains_are_built() { + assertNotNull(getChain("myChain")); + } + + @Test + public void filters_outside_chains_are_built() { + Filter outerFilter = (Filter)http.getFilterChains().componentsRegistry().getComponent("outer"); + assertNotNull(outerFilter); + } + + @Test + public void filters_in_chains_are_built() { + Filter filter = first(getChain("myChain").getInnerComponents()); + assertNotNull(filter); + assertThat(filter.getComponentId().getName(), is("inner")); + } + + private Chain<Filter> getChain(String chainName) { + return http.getFilterChains().allChains().getComponent(ComponentSpecification.fromString(chainName)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java new file mode 100644 index 00000000000..c050b03f25e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterConfigTest.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.core.http.HttpFilterConfig; +import com.yahoo.vespa.model.container.http.xml.HttpBuilder; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import static com.yahoo.collections.CollectionUtil.first; +import static com.yahoo.vespa.model.container.http.FilterConfigProvider.configProviderId; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @since 5.1.23 + */ +public class FilterConfigTest extends DomBuilderTest { + + private Http http; + + @Before + public void setupFilterChains() { + http = new HttpBuilder().build(root, servicesXml()); + root.freezeModelTopology(); + } + + private Element servicesXml() { + return parse( + "<http>", + " <filtering>", + " <filter id='no-config' />", + + " <filter id='empty-config' class='EmptyConfigFilter'>", + " <filter-config />", + " </filter>", + + " <filter id='config-with-params'>", + " <filter-config>", + " <key1>value1</key1>", + " </filter-config>", + " </filter>", + + " <request-chain id='myChain'>", + " <filter id='inner-with-empty-config'>", + " <filter-config />", + " </filter>", + " </request-chain>", + " </filtering>", + "</http>"); + } + + @Test + public void filter_without_config_does_not_have_FilterConfigProvider() { + Filter noConfigFilter = getOuterFilter("no-config"); + + assertThat(getProvider(noConfigFilter), nullValue()); + } + + @Test + public void filterName_is_id_from_component_spec() { + Filter emptyConfigFilter = getOuterFilter("empty-config"); + HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); + + assertThat(config.filterName(), is("empty-config")); + } + + @Test + public void filterClass_is_class_from_component_spec() { + Filter emptyConfigFilter = getOuterFilter("empty-config"); + HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); + + assertThat(config.filterClass(), is("EmptyConfigFilter")); + } + + @Test + public void filter_with_empty_config_has_FilterConfigProvider_with_empty_map() { + Filter emptyConfigFilter = getOuterFilter("empty-config"); + HttpFilterConfig config = getHttpFilterConfig(emptyConfigFilter); + + assertThat(config.param(), is(empty())); + } + + @Test + public void config_params_are_set_correctly_in_FilterConfigProvider() { + Filter configWithParamsFilter = getOuterFilter("config-with-params"); + HttpFilterConfig config = getHttpFilterConfig(configWithParamsFilter); + + assertThat(config.param(), hasSize(1)); + assertThat(config.param(0).name(), is("key1")); + assertThat(config.param(0).value(), is("value1")); + } + + @Test + public void inner_filter_can_have_filter_config() { + Filter innerFilter = (Filter) + first(http.getFilterChains().allChains().getComponent("myChain").getInnerComponents()); + + getHttpFilterConfig(innerFilter); + } + + private Filter getOuterFilter(String id) { + return (Filter)http.getFilterChains().componentsRegistry().getComponent(id); + } + + private static HttpFilterConfig getHttpFilterConfig(Filter filter) { + FilterConfigProvider configProvider = getProvider(filter); + + HttpFilterConfig.Builder builder = new HttpFilterConfig.Builder(); + configProvider.getConfig(builder); + return new HttpFilterConfig(builder); + } + + static FilterConfigProvider getProvider(Filter filter) { + String providerId = configProviderId(filter.getComponentId()).stringValue(); + return (FilterConfigProvider)filter.getChildren().get(providerId); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/processing/test/ProcessingChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/processing/test/ProcessingChainsTest.java new file mode 100644 index 00000000000..f57c2bb9a3c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/processing/test/ProcessingChainsTest.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.processing.test; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.component.chain.Chains; +import com.yahoo.vespa.model.container.processing.ProcessingChain; +import com.yahoo.vespa.model.container.processing.Processor; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Collection; + +import static junit.framework.TestCase.assertEquals; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author bratseth + * @author gjoranv + */ +public class ProcessingChainsTest extends DomBuilderTest { + + private Chains<ProcessingChain> processingChains; + + @Before + public void setupProcessingChains() { + DomProcessingBuilder processingBuilder = new DomProcessingBuilder(null); + processingBuilder.build(root, servicesXml()); + processingChains = (Chains<ProcessingChain>)root.getChildren().get("processing"); + } + + private Element servicesXml() { + return parse( + "<processing>", + " <processor id='processor1' class='com.yahoo.test.Processor1' />", + " <renderer id='renderer1' class='com.yahoo.renderer.Renderer'/>", + " <chain id='default'>", + " <processor idref='processor1'/>", + " <processor id='processor2' class='com.yahoo.test.Processor2'/>", + " </chain>", + "</processing>"); + } + + @Test + public void testProcessingChainConfiguration() { + ProcessingChain defaultChain = processingChains.allChains().getComponent("default"); + assertEquals("default", defaultChain.getId().stringValue()); + assertEquals(1, defaultChain.getInnerComponents().size()); + + Collection<ChainedComponent<?>> outerProcessors = processingChains.getComponentGroup().getComponents(); + assertThat(outerProcessors.size(), is(1)); + assertEquals("processor1", outerProcessors.iterator().next().getComponentId().toString()); + + Collection<Processor> innerProcessors = defaultChain.getInnerComponents(); + assertEquals("processor2", innerProcessors.iterator().next().getComponentId().toString()); + } + + @Test + public void require_that_processors_have_correct_class() { + ChainedComponent<?> processor1 = processingChains.getComponentGroup().getComponents().iterator().next(); + assertThat(processor1.model.bundleInstantiationSpec.classId.stringValue(), + is("com.yahoo.test.Processor1")); + + ProcessingChain defaultChain = processingChains.allChains().getComponent("default"); + Processor processor2 = defaultChain.getInnerComponents().iterator().next(); + assertThat(processor2.model.bundleInstantiationSpec.classId.stringValue(), + is("com.yahoo.test.Processor2")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java new file mode 100644 index 00000000000..209e3791b16 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleImporter; +import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.prelude.semantics.parser.ParseException; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class SemanticRulesTest { + + private final static String root = "src/test/java/com/yahoo/vespa/model/container/search/semanticrules"; + + @Test + public void semanticRulesTest() throws ParseException, IOException { + SemanticRuleBuilder ruleBuilder = new SemanticRuleBuilder(); + SemanticRules rules = ruleBuilder.build(FilesApplicationPackage.fromFile(new File(root))); + SemanticRulesConfig.Builder configBuilder = new SemanticRulesConfig.Builder(); + rules.getConfig(configBuilder); + SemanticRulesConfig config = new SemanticRulesConfig(configBuilder); + Map<String, RuleBase> ruleBases = toMap(config); + assertEquals(2, ruleBases.size()); + assertTrue(ruleBases.containsKey("common")); + assertTrue(ruleBases.containsKey("other")); + assertFalse(ruleBases.get("common").isDefault()); + assertTrue(ruleBases.get("other").isDefault()); + } + + private static Map<String, RuleBase> toMap(SemanticRulesConfig config) throws ParseException, IOException { + RuleImporter ruleImporter = new RuleImporter(config); + Map<String, RuleBase> ruleBaseMap = new HashMap<>(); + for (SemanticRulesConfig.Rulebase ruleBaseConfig : config.rulebase()) { + RuleBase ruleBase = ruleImporter.importConfig(ruleBaseConfig); + if (ruleBaseConfig.isdefault()) + ruleBase.setDefault(true); + ruleBaseMap.put(ruleBase.getName(), ruleBase); + } + return ruleBaseMap; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/Federation2Test.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/Federation2Test.java new file mode 100644 index 00000000000..bc7197b5408 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/Federation2Test.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.test.SimpletypesConfig; +import org.junit.Test; +import org.w3c.dom.Element; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Federation2Test extends SearchChainsTestBase { + @Override + Element servicesXml() { + return parse( + " <search>\n" + + "\n" + + " <chain id=\"chain1\">\n" + + " <searcher id=\"com.yahoo.example.TestSearcher\">\n" + + " <config name=\"test.simpletypes\">\n" + + " <stringval>testSearcher</stringval>\n" + + " </config>\n" + + " </searcher>\n" + + " </chain>\n" + + "\n" + + " <provider id=\"test-source-inherits\">\n" + + " <searcher id=\"com.yahoo.example.AddHitSearcher\" />\n" + + " <source id=\"test-inherits\" />\n" + + " </provider>\n" + + "\n" + + " <!-- Two providers with a common source -->\n" + + " <provider id=\"providerA\">\n" + + " <source id=\"commonSource\">\n" + + " <searcher id=\"com.yahoo.example.AddHitSearcher\">\n" + + " <config name=\"test.simpletypes\">\n" + + " <stringval>providerA</stringval>\n" + + " </config>\n" + + " </searcher>\n" + + " </source>\n" + + " </provider>\n" + + "\n" + + " <provider id=\"providerB\">\n" + + " <source idref=\"commonSource\">\n" + + " <searcher id=\"com.yahoo.example.AddHitSearcher\">\n" + + " <config name=\"test.simpletypes\">\n" + + " <stringval>providerB</stringval>\n" + + " </config>\n" + + " </searcher>\n" + + " </source>\n" + + " </provider>\n" + + "\n" + + " </search>\n"); + } + + + @Test + public void testProviderConfigs() { + //SimpletypesConfig testConfig = root.getConfig(SimpletypesConfig.class, "test/searchchains/chain/chain1/component/com.yahoo.example.TestSearcher"); + //assertEquals("testSearcher",testConfig.stringval()); + + SimpletypesConfig configA = root.getConfig(SimpletypesConfig.class, "searchchains/chain/providerA/source/commonSource/component/com.yahoo.example.AddHitSearcher"); + assertEquals("providerA",configA.stringval()); + + SimpletypesConfig configB = root.getConfig(SimpletypesConfig.class, "searchchains/chain/providerB/source/commonSource/component/com.yahoo.example.AddHitSearcher"); + assertEquals("providerB",configB.stringval()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java new file mode 100644 index 00000000000..d25af7302de --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationTest.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.search.federation.FederationConfig; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * Test generated config for federation. + * @author tonytv + */ +public class FederationTest extends SearchChainsTestBase { + @Override + Element servicesXml() { + return parse( + "<searchchains>", + " <searchchain id='federation1'>", + " <federation id='federationSearcher1'>", + " <source id='source1'>", + " <federationoptions optional='false' />", + " </source>", + " </federation>", + " </searchchain>", + + + " <provider id='provider1'>", + " <federationoptions optional='true' timeout='2.3 s' />", + + " <source id='source1'>", + " <federationoptions timeout='12 ms' />", + " </source>", + " <source id='source2' />", + " <source id='sourceCommon' />", + " </provider>", + + " <provider id='provider2' type='local' cluster='cluster1' />", + + " <provider id='provider3'>", + " <source idref='sourceCommon' />", + " </provider>", + + " <searchchain id='parentChain1' />", + "</searchchains>"); + } + + + @Test + public void validateNativeDefaultTargets() { + FederationConfig.Builder fb = new FederationConfig.Builder(); + root.getConfig(fb, "searchchains/chain/native/component/federation"); + FederationConfig config = new FederationConfig(fb); + + for (FederationConfig.Target target : config.target()) { + String failMessage = "Failed for target " + target.id(); + + if (target.id().startsWith("source")) { + assertTrue(failMessage, target.useByDefault()); + } else { + assertFalse(failMessage, target.useByDefault()); + } + } + + assertThat(config.target().size(), is(5)); + assertUseByDefault(config, "source1", false); + assertUseByDefault(config, "source2", false); + + assertUseByDefault(config, "provider2", true); + assertUseByDefault(config, "cluster2", true); + + assertUseByDefault(config, "sourceCommon", "provider1", false); + assertUseByDefault(config, "sourceCommon", "provider3", false); + + } + + private void assertUseByDefault(FederationConfig config, String sourceName, String providerName, + boolean expectedValue) { + + FederationConfig.Target target = getTarget(config.target(), sourceName); + FederationConfig.Target.SearchChain searchChain = getProvider(target, providerName); + assertThat(searchChain.useByDefault(), is(expectedValue)); + } + + private FederationConfig.Target.SearchChain getProvider(FederationConfig.Target target, String providerName) { + for (FederationConfig.Target.SearchChain searchChain : target.searchChain()) { + if (searchChain.providerId().equals(providerName)) + return searchChain; + } + fail("No provider " + providerName); + return null; + } + + private void assertUseByDefault(FederationConfig config, String chainName, boolean expectedValue) { + FederationConfig.Target target = getTarget(config.target(), chainName); + assertThat(target.searchChain().size(), is(1)); + assertThat(target.searchChain().get(0).useByDefault(), is(expectedValue)); + } + + private FederationConfig.Target getTarget(List<FederationConfig.Target> targets, String chainId) { + for (FederationConfig.Target target : targets) { + if (target.id().equals(chainId)) + return target; + } + fail("No target with id " + chainId); + return null; + } + + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java new file mode 100644 index 00000000000..daef1b845f7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import java.util.HashMap; +import java.util.Map; + +public class MockSearchClusters { + private static class MockSearchCluster extends AbstractSearchCluster { + public MockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { + super(root, clusterName, clusterIndex); + streaming = isStreaming; + } + private final boolean streaming; + + @Override + public int getRowBits() { + return 0; + } + + @Override + protected AbstractSearchCluster.IndexingMode getIndexingMode() { return streaming ? AbstractSearchCluster.IndexingMode.STREAMING : AbstractSearchCluster.IndexingMode.REALTIME; } + @Override + protected void assureSdConsistent() {} + + @Override + public void getConfig(DocumentdbInfoConfig.Builder builder) { + } + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + } + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + } + @Override + public void getConfig(AttributesConfig.Builder builder) { + } + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + } + } + + public static AbstractSearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { + + return new MockSearchCluster(root, clusterName, clusterIndex, isStreaming); + } + + public static Map<String, AbstractSearchCluster> twoMockClusterSpecsByName(AbstractConfigProducerRoot root) { + Map<String, AbstractSearchCluster> result = new HashMap<>(); + result.put("cluster1", mockSearchCluster(root, "cluster1", 1, false)); + result.put("cluster2", mockSearchCluster(root, "cluster2", 2, true)); + return result; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java new file mode 100644 index 00000000000..09c1043734e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.prelude.cluster.ClusterSearcher; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.vespa.defaults.Defaults; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + + +/** + * Test of search chains config + * <p>TODO: examine the actual values in the configs.</p> + * @author tonytv + */ +public class SearchChainsTest extends SearchChainsTestBase { + private ChainsConfig chainsConfig; + private ProviderConfig providerConfig; + private ClusterConfig clusterConfig; + + @Before + public void subscribe() { + ChainsConfig.Builder chainsBuilder = new ChainsConfig.Builder(); + chainsBuilder = (ChainsConfig.Builder)root.getConfig(chainsBuilder, "searchchains"); + chainsConfig = new ChainsConfig(chainsBuilder); + + ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); + providerBuilder = (ProviderConfig.Builder)root.getConfig(providerBuilder, "searchchains/chain/provider:1/component/com.yahoo.search.federation.vespa.VespaSearcher"); + providerConfig = new ProviderConfig(providerBuilder); + + ClusterConfig.Builder clusterBuilder = new ClusterConfig.Builder(); + clusterBuilder = (ClusterConfig.Builder)root.getConfig(clusterBuilder, "searchchains/chain/cluster2/component/" + ClusterSearcher.class.getName()); + clusterConfig = new ClusterConfig(clusterBuilder); + } + + + @Override + Element servicesXml() { + return parse( + "<searchchains>", + " <searcher id='searcher:1' classId='classId1' />", + + " <provider id='provider:1' type='vespa' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", + " cacheweight='2.3'>", + " <federationoptions optional='true' timeout='2.3 s' />", + " <nodes>", + " <node host='sourcehost1' port='12'/>", + " <node host='sourcehost2' port='34'/>", + " </nodes>", + + " <source id='source:1' inherits='parentChain3 parentChain4' excludes='ExcludedSearcher3 ExcludedSearcher4'>", + " <federationoptions timeout='12 ms' />", + " </source>", + " <source id='source:2' />", + " </provider>", + + " <provider id='provider:2' type='local' cluster='cluster1' />", + " <provider id='provider:3' />", + + " <provider id='vespa-provider' type='vespa' >", + " <nodes>", + " <node host='localhost' port='" + Defaults.getDefaults().vespaWebServicePort() + "' />", + " </nodes>", + " <config name='search.federation.provider'>", + " <queryType>PROGRAMMATIC</queryType>", + " </config>", + " </provider>", + + " <searchchain id='default:99'>", + " <federation id='federation:98' provides='provide_federation' before='p1 p2' after='s1 s2'>", + " <source id='source:1'>", + " <federationoptions optional='false' />", + " </source>", + " </federation>", + " </searchchain>", + + " <searchchain id='parentChain1' />", + " <searchchain id='parentChain2' />", + " <searchchain id='parentChain3' />", + " <searchchain id='parentChain4' />", + "</searchchains>"); + } + + @Test + public void require_vespa_searcher_inside_vespa_provider() { + SearchChains searchchains = getSearchChains(); + SearchChain vespaProvider = searchchains.allChains().getComponent("vespa-provider"); + Searcher<?> vespaSearcher = vespaProvider.getInnerComponents().iterator().next(); + assertThat(vespaSearcher, instanceOf(HttpProviderSearcher.class)); + } + + private SearchChains getSearchChains() { + return (SearchChains) root.getChildren().get("searchchains"); + } + + @Test + public void require_user_config_for_vespa_searcher_works() { + assertEquals(root.getConfig(ProviderConfig.class, "searchchains/chain/vespa-provider/component/com.yahoo.search.federation.vespa.VespaSearcher"). + queryType(), ProviderConfig.QueryType.PROGRAMMATIC); + } + + @Test + public void require_that_source_chain_spec_id_is_namespaced_in_provider_id() { + Source source = (Source) getSearchChains().allChains().getComponent("source:1@provider:1"); + assertThat(source.getChainSpecification().componentId.getNamespace(), is(ComponentId.fromString("provider:1"))); + } + + @Test + public void validateHttpProviderConfig() { + assertNotNull(providerConfig); + } + + @Test + public void validateLocalProviderConfig() { + assertEquals(2, clusterConfig.clusterId()); + assertEquals("cluster2", clusterConfig.clusterName()); + } + + public static boolean verifyChainExists(List<ChainsConfig.Chains> chains, String componentId) { + for (ChainsConfig.Chains c : chains) { + if (c.id().equals(componentId)) return true; + } + return false; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest2.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest2.java new file mode 100644 index 00000000000..5e93233f443 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest2.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilderTest; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.hamcrest.Matchers.containsString; + +/** + * @author gjoranv + * @since 5.1.11 + */ +public class SearchChainsTest2 { + + private MockRoot root; + + @Before + public void prepareTest() throws Exception { + root = new MockRoot("root"); + } + + @Test + public void fail_upon_unresolved_inheritance() { + final Element searchElem = DomBuilderTest.parse( + "<search>", + " <chain id='default' inherits='nonexistent' />", + "</search>"); + try { + SearchChains chains = new DomSearchChainsBuilder().build(new MockRoot(), searchElem); + chains.validate(); + fail("Expected exception when inheriting a nonexistent search chain."); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Missing chain 'nonexistent'")); + } + } + + @Test + public void fail_upon_two_user_declared_chains_with_same_name() { + final Element clusterElem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + ContainerModelBuilderTest.nodesXml, + " <search>", + " <chain id='same' />", + " <chain id='same' />", + " </search>", + "</jdisc>"); + try { + ContainerModelBuilderTest.createModel(root, clusterElem); + fail("Expected exception when declaring chains with duplicate id."); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Two entities have the same component id 'same'")); + } + } + + @Test + public void fail_upon_user_declared_chain_with_same_id_as_builtin_chain() throws Exception { + final Element clusterElem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + ContainerModelBuilderTest.nodesXml, + " <search>", + " <chain id='vespa' />", + " </search>", + "</jdisc>"); + try { + ContainerModelBuilderTest.createModel(root, clusterElem); + fail("Expected exception when taking the id from a builtin chain."); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Two entities have the same component id 'vespa'")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTestBase.java new file mode 100644 index 00000000000..a942e428be0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTestBase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder; +import org.junit.Before; +import org.w3c.dom.Element; + +/** Creates SearchChains model from xml input. + * @author tonytv + */ +public abstract class SearchChainsTestBase extends DomBuilderTest { + + @Before + public void setupSearchChains() { + SearchChains searchChains = new DomSearchChainsBuilder().build(root, servicesXml()); + searchChains.initialize(MockSearchClusters.twoMockClusterSpecsByName(root), + new BinaryScaledAmount(100, BinaryPrefix.mega)); + root.freezeModelTopology(); + } + + abstract Element servicesXml(); +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java new file mode 100644 index 00000000000..bf50a40fd2d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupTest.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.hamcrest.Matchers.containsString; + +/** + * @author tonytv + */ +public class SourceGroupTest { + private MockRoot root; + private SearchChains searchChains; + + @Before + public void setUp() throws Exception { + root = new MockRoot(); + searchChains = new SearchChains(root, "searchchains"); + } + + @Test + public void report_error_when_no_leader() { + try { + Provider provider = createProvider("p1"); + Source source = createSource("s1", Source.GroupOption.participant); + provider.addSource(source); + + searchChains.add(provider); + root.freezeModelTopology(); + + searchChains.validate(); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Missing leader for the source s1.")); + return; + } + fail("Expected exception"); + } + + private Provider createProvider(String p1) { + return new Provider(createSearchChainSpecification(p1), new FederationOptions()); + } + + private ChainSpecification createSearchChainSpecification(String id) { + return new ChainSpecification(ComponentId.fromString(id), + new ChainSpecification.Inheritance(null, null), + Collections.<Phase>emptyList(), + Collections.<ComponentSpecification>emptySet()); + } + + private Source createSource(String sourceId, Source.GroupOption groupOption) { + return new Source( + createSearchChainSpecification(sourceId), + new FederationOptions(), + groupOption); + } + + @Test + public void require_that_source_and_provider_id_is_not_allowed_to_be_equal() { + Provider provider = createProvider("sameId"); + Provider provider2 = createProvider("ignoredId"); + + Source source = createSource("sameId", Source.GroupOption.leader); + + provider2.addSource(source); + + searchChains.add(provider); + searchChains.add(provider2); + root.freezeModelTopology(); + + try { + searchChains.validate(); + fail("Expected exception"); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Same id used for a source")); + assertThat(e.getMessage(), containsString("'sameId'")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/common.sr b/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/common.sr new file mode 100644 index 00000000000..d059d3c2de6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/common.sr @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## Some test rules + +# Spelling correction +bahc -> bach; + +# Stopwords +somelongstopword -> ; +[stopword] -> ; +[stopword] :- someotherlongstopword, yetanotherstopword; + + +[song] by [artist] -> song:[song] artist:[artist]; + +[song] :- together, imagine, tinseltown; +[artist] :- youngbloods, beatles, zappa; + +# Negative +various +> -kingz; + + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/other.sr b/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/other.sr new file mode 100644 index 00000000000..b57097287ce --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/semanticrules/rules/other.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@default +@include(common.sr) + +[stopword] -> ; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/PageTemplatesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/PageTemplatesTestCase.java new file mode 100644 index 00000000000..4e8b691518a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/PageTemplatesTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.test; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.model.container.search.PageTemplates; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @author bratseth + */ +public class PageTemplatesTestCase extends junit.framework.TestCase { + + private final static String root="src/test/java/com/yahoo/vespa/model/container/search/test/pages"; + + public void testExport() throws IOException { + List<NamedReader> pageFiles=new ArrayList<>(2); + pageFiles.add(new NamedReader(root + "/slottingSerp.xml", IOUtils.createReader(root + "/slottingSerp.xml"))); + pageFiles.add(new NamedReader(root + "/richSerp.xml", IOUtils.createReader(root + "/richSerp.xml"))); + pageFiles.add(new NamedReader(root + "/footer.xml", IOUtils.createReader(root + "/footer.xml"))); + pageFiles.add(new NamedReader(root + "/richerSerp.xml", IOUtils.createReader(root + "/richerSerp.xml"))); + pageFiles.add(new NamedReader(root + "/header.xml", IOUtils.createReader(root + "/header.xml"))); + assertEquals(IOUtils.readFile(new File(root, "/pages.cfg")), StringUtilities.implodeMultiline(ConfigInstance.serialize(new PageTemplates(pageFiles).getConfig()))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfileVariantsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfileVariantsTestCase.java new file mode 100644 index 00000000000..6761bd9bef7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfileVariantsTestCase.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.test; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.io.IOException; + +import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; + +/** + * @author bratseth + */ +public class QueryProfileVariantsTestCase extends junit.framework.TestCase { + + private final String root = "src/test/java/com/yahoo/vespa/model/container/search/test/"; + + public void testConfigCreation() throws IOException { + QueryProfileRegistry registry = new QueryProfileXMLReader().read(root + "queryprofilevariants"); + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "query-profile-variants-configuration.cfg", profiles.getConfig().toString()); + } + + public void testConfigCreation2() throws IOException { + QueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2"); + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "query-profile-variants2-configuration.cfg", profiles.getConfig().toString()); + } + + public void testConfigCreationNewsBESimple() throws IOException { + QueryProfileRegistry registry = new QueryProfileXMLReader().read(root + "newsbesimple"); + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "newsbe-query-profiles-simple.cfg", profiles.getConfig().toString()); + } + + public void testConfigCreationNewsFESimple() throws IOException { + QueryProfileRegistry registry = new QueryProfileXMLReader().read(root + "newsfesimple"); + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "newsfe-query-profiles-simple.cfg", profiles.getConfig().toString()); + } + + public void testVariantsOfExplicitCompound() throws IOException { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile a1 = new QueryProfile("a1"); + a1.set("b", "a1.b", registry); + + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a", a1, registry); + profile.set("a.b", "a.b.x1", new String[] {"x1"}, registry); + profile.set("a.b", "a.b.x2", new String[] {"x2"}, registry); + + registry.register(a1); + registry.register(profile); + + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "variants-of-explicit-compound.cfg", profiles.getConfig().toString()); + } + + public void testVariantsOfExplicitCompoundWithVariantReference() throws IOException { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile a1 = new QueryProfile("a1"); + a1.set("b", "a1.b", registry); + + QueryProfile a2 = new QueryProfile("a2"); + a2.set("b", "a2.b", registry); + + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a", a1, registry); + profile.set("a", a2, new String[] {"x1"}, registry); + profile.set("a.b", "a.b.x1", new String[] {"x1"}, registry); + profile.set("a.b", "a.b.x2", new String[] {"x2"}, registry); + + registry.register(a1); + registry.register(a2); + registry.register(profile); + + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "variants-of-explicit-compound-with-reference.cfg", profiles.getConfig().toString()); + } + + /** For comparison with the above */ + public void testExplicitReferenceOverride() throws IOException { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile a1 = new QueryProfile("a1"); + a1.set("b", "a1.b", registry); + + QueryProfile profile = new QueryProfile("test"); + profile.set("a", a1, registry); + profile.set("a.b", "a.b", registry); + assertEquals("a.b", profile.get("a.b")); + + registry.register(a1); + registry.register(profile); + + QueryProfiles profiles = new QueryProfiles(registry); + assertSerializedConfigFileEquals(root + "explicit-reference-override.cfg", profiles.getConfig().toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java new file mode 100644 index 00000000000..b9ec84c389b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -0,0 +1,122 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; + +/** + * Tests generation of config from query profiles (XML reading is tested elsewhere) + * + * @author bratseth + */ +public class QueryProfilesTestCase extends junit.framework.TestCase { + + private final static String root="src/test/java/com/yahoo/vespa/model/container/search/test/"; + + public void testEmpty() throws IOException { + QueryProfileRegistry reg = new QueryProfileRegistry(); + assertConfig("empty.cfg", reg); + } + + public void testQueryProfiles() throws IOException { + final boolean mandatory=true; + final boolean overridable=true; + QueryProfileRegistry registry=new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry=registry.getTypeRegistry(); + + QueryProfileType userType=new QueryProfileType("user"); + userType.setStrict(true); + userType.addField(new FieldDescription("robot", FieldType.fromString("boolean",typeRegistry), "machine automaton", mandatory, !overridable)); + userType.addField(new FieldDescription("ads", FieldType.fromString("string",typeRegistry), mandatory, overridable)); + userType.addField(new FieldDescription("age", FieldType.fromString("integer",typeRegistry), !mandatory, overridable)); + typeRegistry.register(userType); + + QueryProfileType rootType=new QueryProfileType("root"); + QueryProfileType nativeProfile=typeRegistry.getComponent("native"); + assertNotNull(nativeProfile); + assertTrue(nativeProfile.isBuiltin()); + rootType.inherited().add(nativeProfile); + rootType.setMatchAsPath(true); + rootType.addField(new FieldDescription("user", FieldType.fromString("query-profile:user",typeRegistry), mandatory, overridable)); + typeRegistry.register(rootType); + + QueryProfileType marketType=new QueryProfileType("market"); + marketType.inherited().add(rootType); + marketType.addField(new FieldDescription("market", FieldType.fromString("string",typeRegistry), !mandatory, !overridable)); + typeRegistry.register(marketType); + + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.set("ranking","production23", registry); + defaultProfile.set("representation.defaultIndex", "title", registry); + defaultProfile.setOverridable("representation.defaultIndex", false, null); + registry.register(defaultProfile); + + QueryProfile test=new QueryProfile("test"); + test.set("tracelevel",2,registry); + registry.register(test); + + QueryProfile genericUser=new QueryProfile("genericUser"); + genericUser.setType(userType); + genericUser.set("robot",false,registry); + genericUser.set("ads","all",registry); + registry.register(genericUser); + + QueryProfile root=new QueryProfile("root"); + root.setType(rootType); + root.addInherited(defaultProfile); + root.addInherited(test); + root.set("hits",30,registry); + root.setOverridable("hits",false,null); + root.set("unique","category",registry); + root.set("user",genericUser,registry); + root.set("defaultage", "7d",registry); + registry.register(root); + + QueryProfile marketUser=new QueryProfile("marketUser"); + marketUser.setType(userType); + marketUser.addInherited(genericUser); + marketUser.set("ads","none",registry); + marketUser.set("age",25,registry); + registry.register(marketUser); + + QueryProfile market=new QueryProfile("root/market"); + market.setType(marketType); + market.addInherited(root); + market.set("hits",15,registry); + market.set("user",marketUser,registry); + market.set("market","some market",registry); + market.set("marketHeading","Market of %{market}",registry); + registry.register(market); + + QueryProfile untypedUser=new QueryProfile("untypedUser"); + untypedUser.set("robot",false,registry); + untypedUser.set("robot.type","continent-class",registry); + registry.register(untypedUser); + + assertConfig("query-profiles.cfg",registry); + } + + protected void assertConfig(String correctFileName, QueryProfileRegistry check) throws IOException { + assertSerializedConfigFileEquals(root + "/" + correctFileName, + com.yahoo.text.StringUtilities.implodeMultiline(com.yahoo.config.ConfigInstance.serialize(new QueryProfiles(check).getConfig()))); + + // Also assert that the correct config config can actually be read as a config source + QueryProfileConfigurer configurer = new QueryProfileConfigurer("file:" + root + "empty.cfg"); + configurer.shutdown(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg new file mode 100644 index 00000000000..99e2e3c5dcb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg @@ -0,0 +1,13 @@ +queryprofile[0].id "a1" +queryprofile[0].type "" +queryprofile[0].property[0].name "b" +queryprofile[0].property[0].value "a1.b" +queryprofile[0].property[0].overridable "" +queryprofile[1].id "test" +queryprofile[1].type "" +queryprofile[1].property[0].name "a.b" +queryprofile[1].property[0].value "a.b" +queryprofile[1].property[0].overridable "" +queryprofile[1].reference[0].name "a" +queryprofile[1].reference[0].value "a1" +queryprofile[1].reference[0].overridable ""
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg new file mode 100644 index 00000000000..196b6c3513a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg @@ -0,0 +1,19 @@ +queryprofile[0].id "scthumbnail" +queryprofile[0].type "" +queryprofile[0].dimensions[0] "custid_1" +queryprofile[0].dimensions[1] "custid_2" +queryprofile[0].dimensions[2] "custid_3" +queryprofile[0].dimensions[3] "custid_4" +queryprofile[0].dimensions[4] "custid_5" +queryprofile[0].dimensions[5] "custid_6" +queryprofile[0].property[0].name "debug.query.profile.file.scthumbnail.xml" +queryprofile[0].property[0].value "" +queryprofile[0].property[0].overridable "" +queryprofile[0].property[1].name "scthumbnail.activate" +queryprofile[0].property[1].value "true" +queryprofile[0].property[1].overridable "true" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[0] "yahoo" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[1] "uk" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[2] "sc" +queryprofile[0].queryprofilevariant[0].property[0].name "scthumbnail.sourcecountry" +queryprofile[0].queryprofilevariant[0].property[0].value "uk"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbesimple/scthumbnail.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbesimple/scthumbnail.xml new file mode 100644 index 00000000000..f3d1e6892ba --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbesimple/scthumbnail.xml @@ -0,0 +1,14 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="scthumbnail"> + <dimensions>custid_1,custid_2,custid_3,custid_4,custid_5,custid_6</dimensions> + + <field name="debug.query.profile.file.scthumbnail.xml"/> + <field name="scthumbnail.activate" overridable="true">true</field> + + <!-- Set the source country used for the thumb query, default US --> + <query-profile for="yahoo,uk,sc"> + <field name="scthumbnail.sourcecountry">uk</field> + </query-profile> +</query-profile> + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg new file mode 100644 index 00000000000..461f9b606c6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg @@ -0,0 +1,26 @@ +queryprofile[0].id "backend/news" +queryprofile[0].type "" +queryprofile[0].dimensions[0] "vertical" +queryprofile[0].dimensions[1] "sort" +queryprofile[0].dimensions[2] "offset" +queryprofile[0].dimensions[3] "resulttypes" +queryprofile[0].dimensions[4] "rss" +queryprofile[0].dimensions[5] "age" +queryprofile[0].dimensions[6] "intl" +queryprofile[0].dimensions[7] "testid" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[0] "news" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[1] "*" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[2] "*" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[3] "article" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[4] "0" +queryprofile[0].queryprofilevariant[0].property[0].name "discovery" +queryprofile[0].queryprofilevariant[0].property[0].value "sources" +queryprofile[0].queryprofilevariant[0].property[1].name "discovery.sources.count" +queryprofile[0].queryprofilevariant[0].property[1].value "13" +queryprofile[0].queryprofilevariant[0].property[2].name "discoverytypes" +queryprofile[0].queryprofilevariant[0].property[2].value "article" +queryprofile[1].id "default" +queryprofile[1].type "" +queryprofile[1].reference[0].name "source.news" +queryprofile[1].reference[0].value "backend/news" +queryprofile[1].reference[0].overridable ""
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/backend_news.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/backend_news.xml new file mode 100644 index 00000000000..3585ccd5eda --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/backend_news.xml @@ -0,0 +1,9 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="backend/news"> + <dimensions>vertical,sort,offset,resulttypes,rss,age,intl,testid</dimensions> + <query-profile for="news,*,*,article,0"> + <field name="discovery">sources</field> + <field name="discoverytypes">article</field> + <field name="discovery.sources.count">13</field> + </query-profile> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/default.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/default.xml new file mode 100644 index 00000000000..d8dbe6e929a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfesimple/default.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <field name="source.news"><ref>backend/news</ref></field> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/footer.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/footer.xml new file mode 100644 index 00000000000..0866aaaa583 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/footer.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="footer"> + <section layout="row" source="popularSearches"/> + <section id="extraFooter" layout="row" source="topArticles"/> +</page> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/header.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/header.xml new file mode 100644 index 00000000000..a894e8b9a3e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/header.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="header"> + <section layout="row"> + <section source="global"/> + <section source="notifications"/> + </section> +</page> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/pages.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/pages.cfg new file mode 100644 index 00000000000..a65903d052b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/pages.cfg @@ -0,0 +1,5 @@ +page[0] "<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->\n<page id=\"slottingSerp\" layout=\"mainAndRight\">\n <section layout=\"column\" placement=\"main\" source=\"*\" blending=\"slot\"/>\n <section layout=\"column\" placement=\"right\" source=\"ads\"/>\n</page>\n" +page[1] "<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->\n<page id=\"richSerp\" layout=\"mainAndRight\">\n <section layout=\"row\" placement=\"main\">\n <section layout=\"column\" description=\"left main pane\">\n <section layout=\"row\" max=\"5\" description=\"Bar of images, from one of two possible sources\">\n <choice>\n <source name=\"images\"/>\n <source name=\"flickr\"/>\n </choice>\n </section>\n <section max=\"1\" source=\"local map video ticker weather\" description=\"A single relevant graphically rich element\"/>\n <section blending=\"blend\" max=\"10\" source=\"web news\" description=\"Various kinds of traditional search results\"/>\n </section>\n <section layout=\"column\" blending=\"group\" source=\"answers blogs twitter\" description=\"right main pane, ugc stuff, grouped by source\"/>\n </section>\n <section layout=\"column\" source=\"ads\" placement=\"right\"/>\n</page>\n" +page[2] "<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->\n<page id=\"footer\">\n <section layout=\"row\" source=\"popularSearches\"/>\n <section id=\"extraFooter\" layout=\"row\" source=\"topArticles\"/>\n</page>\n" +page[3] "<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->\n<page id=\"richerSerp\" layout=\"column\">\n <include idref=\"header\"/>\n <section layout=\"mainAndRight\">\n <section layout=\"row\" placement=\"main\">\n <section layout=\"column\" description=\"left main pane\">\n <choice>\n <alternative>\n <section layout=\"row\" max=\"5\" description=\"Bar of images, from one of two possible sources\">\n <choice>\n <source name=\"images\"/>\n <alternative>\n <source name=\"flickr\">\n <presentation name=\"mouseOverImage\"/>\n </source>\n <source name=\"twitpic\">\n <choice>\n <presentation name=\"mouseOverImage\">\n <parameter name=\"hovertime\">5</parameter>\n <parameter name=\"borderColor\">#ff00ff</parameter>\n </presentation>\n <presentation name=\"regularImage\"/>\n </choice>\n <parameter name=\"filter\">origin=twitter</parameter>\n </source>\n </alternative>\n </choice>\n <choice>\n <presentation name=\"regularImageBox\"/>\n <presentation name=\"newImageBox\"/>\n </choice>\n </section>\n <section max=\"1\" source=\"local map video ticker weather\" description=\"A single relevant graphically rich element\"/>\n </alternative>\n <section blending=\"blend\" max=\"10\" source=\"web news\" description=\"Various kinds of traditional search results\"/>\n </choice>\n </section>\n <section layout=\"column\" blending=\"group\" source=\"answers blogs twitter\" description=\"right main pane, ugc stuff, grouped by source\"/>\n </section>\n <section layout=\"column\" source=\"ads\" placement=\"right\" order=\"relevance clickProbability\">\n <presentation name=\"newAdBox\"/>\n </section>\n </section>\n <include idref=\"footer\"/>\n</page>\n" +page[4] "<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->\n<page id=\"header\">\n <section layout=\"row\">\n <section source=\"global\"/>\n <section source=\"notifications\"/>\n </section>\n</page>\n"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richSerp.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richSerp.xml new file mode 100644 index 00000000000..f47b57d699e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richSerp.xml @@ -0,0 +1,17 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="richSerp" layout="mainAndRight"> + <section layout="row" placement="main"> + <section layout="column" description="left main pane"> + <section layout="row" max="5" description="Bar of images, from one of two possible sources"> + <choice> + <source name="images"/> + <source name="flickr"/> + </choice> + </section> + <section max="1" source="local map video ticker weather" description="A single relevant graphically rich element"/> + <section blending="blend" max="10" source="web news" description="Various kinds of traditional search results"/> + </section> + <section layout="column" blending="group" source="answers blogs twitter" description="right main pane, ugc stuff, grouped by source"/> + </section> + <section layout="column" source="ads" placement="right"/> +</page> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richerSerp.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richerSerp.xml new file mode 100644 index 00000000000..e2206c0f288 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/richerSerp.xml @@ -0,0 +1,45 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="richerSerp" layout="column"> + <include idref="header"/> + <section layout="mainAndRight"> + <section layout="row" placement="main"> + <section layout="column" description="left main pane"> + <choice> + <alternative> + <section layout="row" max="5" description="Bar of images, from one of two possible sources"> + <choice> + <source name="images"/> + <alternative> + <source name="flickr"> + <presentation name="mouseOverImage"/> + </source> + <source name="twitpic"> + <choice> + <presentation name="mouseOverImage"> + <parameter name="hovertime">5</parameter> + <parameter name="borderColor">#ff00ff</parameter> + </presentation> + <presentation name="regularImage"/> + </choice> + <parameter name="filter">origin=twitter</parameter> + </source> + </alternative> + </choice> + <choice> + <presentation name="regularImageBox"/> + <presentation name="newImageBox"/> + </choice> + </section> + <section max="1" source="local map video ticker weather" description="A single relevant graphically rich element"/> + </alternative> + <section blending="blend" max="10" source="web news" description="Various kinds of traditional search results"/> + </choice> + </section> + <section layout="column" blending="group" source="answers blogs twitter" description="right main pane, ugc stuff, grouped by source"/> + </section> + <section layout="column" source="ads" placement="right" order="relevance clickProbability"> + <presentation name="newAdBox"/> + </section> + </section> + <include idref="footer"/> +</page> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/slottingSerp.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/slottingSerp.xml new file mode 100644 index 00000000000..8e40909a489 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/pages/slottingSerp.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="slottingSerp" layout="mainAndRight"> + <section layout="column" placement="main" source="*" blending="slot"/> + <section layout="column" placement="right" source="ads"/> +</page> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg new file mode 100644 index 00000000000..c86bba23286 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg @@ -0,0 +1,41 @@ +queryprofile[0].id "variants1" +queryprofile[0].type "" +queryprofile[0].dimensions[0] "x" +queryprofile[0].dimensions[1] "y" +queryprofile[0].property[0].name "a" +queryprofile[0].property[0].value "a-deflt" +queryprofile[0].property[0].overridable "" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[0].inherit[0] "variants2" +queryprofile[0].queryprofilevariant[0].inherit[1] "wparent2" +queryprofile[0].queryprofilevariant[0].property[0].name "a" +queryprofile[0].queryprofilevariant[0].property[0].value "x1.y1.a" +queryprofile[0].queryprofilevariant[0].property[1].name "b" +queryprofile[0].queryprofilevariant[0].property[1].value "x1.y1.b" +queryprofile[0].queryprofilevariant[1].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[1].property[0].name "a" +queryprofile[0].queryprofilevariant[1].property[0].value "x1.y?.a" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[0] "*" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[2].property[0].name "a" +queryprofile[0].queryprofilevariant[2].property[0].value "x?.y1.a" +queryprofile[0].queryprofilevariant[2].reference[0].name "toVariants2" +queryprofile[0].queryprofilevariant[2].reference[0].value "variants2" +queryprofile[1].id "variants2" +queryprofile[1].type "" +queryprofile[1].dimensions[0] "x" +queryprofile[1].property[0].name "c" +queryprofile[1].property[0].value "c-df" +queryprofile[1].property[0].overridable "" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[1].queryprofilevariant[0].property[0].name "c" +queryprofile[1].queryprofilevariant[0].property[0].value "x1.c" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "x2" +queryprofile[1].queryprofilevariant[1].property[0].name "c" +queryprofile[1].queryprofilevariant[1].property[0].value "x2.c" +queryprofile[2].id "wparent2" +queryprofile[2].type "" +queryprofile[2].property[0].name "a" +queryprofile[2].property[0].value "a1" +queryprofile[2].property[0].overridable ""
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg new file mode 100644 index 00000000000..c915cd2efd0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg @@ -0,0 +1,61 @@ +queryprofile[0].id "default" +queryprofile[0].type "" +queryprofile[0].property[0].name "hits" +queryprofile[0].property[0].value "5" +queryprofile[0].property[0].overridable "" +queryprofile[0].property[1].name "model.defaultIndex" +queryprofile[0].property[1].value "title" +queryprofile[0].property[1].overridable "" +queryprofile[0].property[2].name "ranking.features.query(scorelimit)" +queryprofile[0].property[2].value "-20" +queryprofile[0].property[2].overridable "" +queryprofile[0].property[3].name "ranking.profile" +queryprofile[0].property[3].value "production1" +queryprofile[0].property[3].overridable "" +queryprofile[0].property[4].name "ranking.properties.dotProduct.X" +queryprofile[0].property[4].value "(a:1,b:2)" +queryprofile[0].property[4].overridable "" +queryprofile[1].id "multi" +queryprofile[1].type "" +queryprofile[1].inherit[0] "default" +queryprofile[1].dimensions[0] "myquery" +queryprofile[1].dimensions[1] "myindex" +queryprofile[1].property[0].name "model.defaultIndex" +queryprofile[1].property[0].value "default-default" +queryprofile[1].property[0].overridable "" +queryprofile[1].reference[0].name "model" +queryprofile[1].reference[0].value "querybest" +queryprofile[1].reference[0].overridable "" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "love" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[0].reference[0].name "model" +queryprofile[1].queryprofilevariant[0].reference[0].value "querylove" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[1].property[0].name "model.defaultIndex" +queryprofile[1].queryprofilevariant[1].property[0].value "default" +queryprofile[1].queryprofilevariant[2].fordimensionvalues[0] "love" +queryprofile[1].queryprofilevariant[2].reference[0].name "model" +queryprofile[1].queryprofilevariant[2].reference[0].value "querylove" +queryprofile[2].id "querybest" +queryprofile[2].type "model" +queryprofile[2].property[0].name "defaultIndex" +queryprofile[2].property[0].value "title" +queryprofile[2].property[0].overridable "" +queryprofile[2].property[1].name "queryString" +queryprofile[2].property[1].value "best" +queryprofile[2].property[1].overridable "false" +queryprofile[3].id "querylove" +queryprofile[3].type "model" +queryprofile[3].dimensions[0] "myquery" +queryprofile[3].dimensions[1] "myindex" +queryprofile[3].property[0].name "defaultIndex" +queryprofile[3].property[0].value "title" +queryprofile[3].property[0].overridable "" +queryprofile[3].property[1].name "queryString" +queryprofile[3].property[1].value "love" +queryprofile[3].property[1].overridable "false" +queryprofile[3].queryprofilevariant[0].fordimensionvalues[0] "love" +queryprofile[3].queryprofilevariant[0].fordimensionvalues[1] "default" +queryprofile[3].queryprofilevariant[0].property[0].name "defaultIndex" +queryprofile[3].queryprofilevariant[0].property[0].value "default"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg new file mode 100644 index 00000000000..89a971adb15 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg @@ -0,0 +1,105 @@ +queryprofile[0].id "default" +queryprofile[0].type "" +queryprofile[0].property[0].name "ranking" +queryprofile[0].property[0].value "production23" +queryprofile[0].property[0].overridable "" +queryprofile[0].property[1].name "representation.defaultIndex" +queryprofile[0].property[1].value "title" +queryprofile[0].property[1].overridable "false" +queryprofile[1].id "test" +queryprofile[1].type "" +queryprofile[1].property[0].name "tracelevel" +queryprofile[1].property[0].value "2" +queryprofile[1].property[0].overridable "" +queryprofile[2].id "genericUser" +queryprofile[2].type "user" +queryprofile[2].property[0].name "ads" +queryprofile[2].property[0].value "all" +queryprofile[2].property[0].overridable "" +queryprofile[2].property[1].name "robot" +queryprofile[2].property[1].value "false" +queryprofile[2].property[1].overridable "" +queryprofile[3].id "root" +queryprofile[3].type "root" +queryprofile[3].inherit[0] "default" +queryprofile[3].inherit[1] "test" +queryprofile[3].property[0].name "defaultage" +queryprofile[3].property[0].value "7d" +queryprofile[3].property[0].overridable "" +queryprofile[3].property[1].name "hits" +queryprofile[3].property[1].value "30" +queryprofile[3].property[1].overridable "false" +queryprofile[3].property[2].name "unique" +queryprofile[3].property[2].value "category" +queryprofile[3].property[2].overridable "" +queryprofile[3].reference[0].name "user" +queryprofile[3].reference[0].value "genericUser" +queryprofile[3].reference[0].overridable "" +queryprofile[4].id "marketUser" +queryprofile[4].type "user" +queryprofile[4].inherit[0] "genericUser" +queryprofile[4].property[0].name "ads" +queryprofile[4].property[0].value "none" +queryprofile[4].property[0].overridable "" +queryprofile[4].property[1].name "age" +queryprofile[4].property[1].value "25" +queryprofile[4].property[1].overridable "" +queryprofile[5].id "root/market" +queryprofile[5].type "market" +queryprofile[5].inherit[0] "root" +queryprofile[5].property[0].name "hits" +queryprofile[5].property[0].value "15" +queryprofile[5].property[0].overridable "" +queryprofile[5].property[1].name "market" +queryprofile[5].property[1].value "some market" +queryprofile[5].property[1].overridable "" +queryprofile[5].property[2].name "marketHeading" +queryprofile[5].property[2].value "Market of %{market}" +queryprofile[5].property[2].overridable "" +queryprofile[5].reference[0].name "user" +queryprofile[5].reference[0].value "marketUser" +queryprofile[5].reference[0].overridable "" +queryprofile[6].id "untypedUser" +queryprofile[6].type "" +queryprofile[6].property[0].name "robot" +queryprofile[6].property[0].value "false" +queryprofile[6].property[0].overridable "" +queryprofile[6].property[1].name "robot.type" +queryprofile[6].property[1].value "continent-class" +queryprofile[6].property[1].overridable "" +queryprofiletype[0].id "user" +queryprofiletype[0].strict true +queryprofiletype[0].matchaspath false +queryprofiletype[0].field[0].name "ads" +queryprofiletype[0].field[0].type "string" +queryprofiletype[0].field[0].overridable true +queryprofiletype[0].field[0].mandatory true +queryprofiletype[0].field[0].alias "" +queryprofiletype[0].field[1].name "age" +queryprofiletype[0].field[1].type "integer" +queryprofiletype[0].field[1].overridable true +queryprofiletype[0].field[1].mandatory false +queryprofiletype[0].field[1].alias "" +queryprofiletype[0].field[2].name "robot" +queryprofiletype[0].field[2].type "boolean" +queryprofiletype[0].field[2].overridable false +queryprofiletype[0].field[2].mandatory true +queryprofiletype[0].field[2].alias "machine automaton" +queryprofiletype[1].id "root" +queryprofiletype[1].strict false +queryprofiletype[1].matchaspath true +queryprofiletype[1].inherit[0] "native" +queryprofiletype[1].field[0].name "user" +queryprofiletype[1].field[0].type "query-profile:user" +queryprofiletype[1].field[0].overridable true +queryprofiletype[1].field[0].mandatory true +queryprofiletype[1].field[0].alias "" +queryprofiletype[2].id "market" +queryprofiletype[2].strict false +queryprofiletype[2].matchaspath false +queryprofiletype[2].inherit[0] "root" +queryprofiletype[2].field[0].name "market" +queryprofiletype[2].field[0].type "string" +queryprofiletype[2].field[0].overridable false +queryprofiletype[2].field[0].mandatory false +queryprofiletype[2].field[0].alias ""
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants1.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants1.xml new file mode 100644 index 00000000000..b4a1398a83e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants1.xml @@ -0,0 +1,16 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="variants1"> + <dimensions>x,y</dimensions> + <field name="a">a-deflt</field> + <query-profile for="x1,y1" inherits="variants2 wparent2"> + <field name="a">x1.y1.a</field> + <field name="b">x1.y1.b</field> + </query-profile> + <query-profile for="x1"> + <field name="a">x1.y?.a</field> + </query-profile> + <query-profile for="*,y1"> + <field name="a">x?.y1.a</field> + <field name="toVariants2"><ref>variants2</ref></field> + </query-profile> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants2.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants2.xml new file mode 100644 index 00000000000..23ce86b07a6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/variants2.xml @@ -0,0 +1,11 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="variants2"> + <dimensions>x</dimensions> + <field name="c">c-df</field> + <query-profile for="x1"> + <field name="c">x1.c</field> + </query-profile> + <query-profile for="x2"> + <field name="c">x2.c</field> + </query-profile> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/wparent2.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/wparent2.xml new file mode 100644 index 00000000000..84ae3bd8c01 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants/wparent2.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="wparent2"> + <field name="a">a1</field> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/default.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/default.xml new file mode 100644 index 00000000000..4ffe97ef9a2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/default.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <field name="hits">5</field> + <field name="model.defaultIndex">title</field> + <field name="ranking.profile">production1</field> + <field name="ranking.features.query(scorelimit)">-20</field> + <field name="ranking.properties.dotProduct.X">(a:1,b:2)</field> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/multi.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/multi.xml new file mode 100644 index 00000000000..0bd52c8f8ee --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/multi.xml @@ -0,0 +1,20 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="multi" inherits="default"> <!-- default sets default-index to title --> + <dimensions>myquery, myindex </dimensions> + <field name="model"><ref>querybest</ref></field> + <field name="model.defaultIndex">default-default</field> + + <query-profile for="love,default"> + <field name="model"><ref>querylove</ref></field> + <field name="model.defaultIndex">default</field> + </query-profile> + + <query-profile for="*,default"> + <field name="model.defaultIndex">default</field> + </query-profile> + + <query-profile for="love"> + <field name="model"><ref>querylove</ref></field> + </query-profile> + +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querybest.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querybest.xml new file mode 100644 index 00000000000..9a957012de4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querybest.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="querybest" type="model"> + <field name="defaultIndex">title</field> + <field name="queryString" overridable="false">best</field> +</query-profile> + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querylove.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querylove.xml new file mode 100644 index 00000000000..e7864977804 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/queryprofilevariants2/querylove.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="querylove" type="model"> + <field name="defaultIndex">title</field> + <field name="queryString" overridable="false">love</field> +</query-profile> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg new file mode 100644 index 00000000000..e1cca7ed232 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg @@ -0,0 +1,26 @@ +queryprofile[0].id "a1" +queryprofile[0].type "" +queryprofile[0].property[0].name "b" +queryprofile[0].property[0].value "a1.b" +queryprofile[0].property[0].overridable "" +queryprofile[1].id "a2" +queryprofile[1].type "" +queryprofile[1].dimensions[0] "x" +queryprofile[1].property[0].name "b" +queryprofile[1].property[0].value "a2.b" +queryprofile[1].property[0].overridable "" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[1].queryprofilevariant[0].property[0].name "b" +queryprofile[1].queryprofilevariant[0].property[0].value "a.b.x1" +queryprofile[2].id "test" +queryprofile[2].type "" +queryprofile[2].dimensions[0] "x" +queryprofile[2].reference[0].name "a" +queryprofile[2].reference[0].value "a1" +queryprofile[2].reference[0].overridable "" +queryprofile[2].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[2].queryprofilevariant[0].reference[0].name "a" +queryprofile[2].queryprofilevariant[0].reference[0].value "a2" +queryprofile[2].queryprofilevariant[1].fordimensionvalues[0] "x2" +queryprofile[2].queryprofilevariant[1].property[0].name "a.b" +queryprofile[2].queryprofilevariant[1].property[0].value "a.b.x2"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg new file mode 100644 index 00000000000..d65b3fa5f92 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg @@ -0,0 +1,17 @@ +queryprofile[0].id "a1" +queryprofile[0].type "" +queryprofile[0].property[0].name "b" +queryprofile[0].property[0].value "a1.b" +queryprofile[0].property[0].overridable "" +queryprofile[1].id "test" +queryprofile[1].type "" +queryprofile[1].dimensions[0] "x" +queryprofile[1].reference[0].name "a" +queryprofile[1].reference[0].value "a1" +queryprofile[1].reference[0].overridable "" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[1].queryprofilevariant[0].property[0].name "a.b" +queryprofile[1].queryprofilevariant[0].property[0].value "a.b.x1" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "x2" +queryprofile[1].queryprofilevariant[1].property[0].name "a.b" +queryprofile[1].queryprofilevariant[1].property[0].value "a.b.x2"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java new file mode 100644 index 00000000000..99f4387c140 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.container.logging.VespaAccessLog; +import com.yahoo.container.logging.YApacheAccessLog; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import org.junit.Test; +import org.w3c.dom.Element; + +import static com.yahoo.text.StringUtilities.quote; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @since 5.5 + */ +public class AccessLogTest extends ContainerModelBuilderTestBase { + + @Test + public void default_access_log_is_only_added_when_search_is_present() throws Exception { + Element cluster1Elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + "<search />", + nodesXml, + "</jdisc>"); + Element cluster2Elem = DomBuilderTest.parse( + "<jdisc id='cluster2' version='1.0'>", + " <nodes>", + " <node hostalias='mockhost' baseport='1234' />", + " </nodes>", + "</jdisc>" ); + + createModel(root, cluster1Elem, cluster2Elem); + + assertNotNull(getVespaAccessLog("cluster1")); + assertNull( getVespaAccessLog("cluster2")); + } + + @Test + public void default_search_access_log_can_be_disabled() throws Exception { + final String jdiscClusterId = "jdisc-cluster"; + + Element clusterElem = DomBuilderTest.parse( + "<jdisc id=" + quote(jdiscClusterId) + " version='1.0'>" + + " <search />" + + " <accesslog type='disabled' />" + + "</jdisc>" ); + + createModel(root, clusterElem); + assertNull(getVespaAccessLog(jdiscClusterId)); + } + + private Component<?, ?> getVespaAccessLog(String clusterName) { + ContainerCluster cluster = (ContainerCluster) root.getChildren().get(clusterName); + return cluster.getComponentsMap().get(ComponentId.fromString((VespaAccessLog.class.getName()))); + + } + + @Test + public void access_log_can_be_configured() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <accesslog type='yapache' ", + " fileNamePattern='pattern' rotationInterval='interval'", + " rotationScheme='date' />", + nodesXml, + "</jdisc>" ); + + createModel(root, clusterElem); + + Component<?, ?> accessLogComponent = getContainerComponent("default", YApacheAccessLog.class.getName()); + assertNotNull(accessLogComponent); + assertThat(accessLogComponent.getClassId().getName(), is(YApacheAccessLog.class.getName())); + + AccessLogConfig config = root.getConfig(AccessLogConfig.class, "default/component/com.yahoo.container.logging.YApacheAccessLog"); + AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); + assertThat(fileHandlerConfig.pattern(), is("pattern")); + assertThat(fileHandlerConfig.rotation(), is("interval")); + assertThat(fileHandlerConfig.rotateScheme(), is(AccessLogConfig.FileHandler.RotateScheme.DATE)); + + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java new file mode 100644 index 00000000000..7a8a554e650 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.cloud.config.ElkConfig; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.configserver.TestOptions; +import org.junit.Test; + +import java.io.File; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.17 + */ +public class ConfigServerContainerModelBuilderTest { + @Test + public void testHostedVespaInclude() { + File testApp = new File("src/test/cfg/container/data/configserverinclude"); + FilesApplicationPackage app = FilesApplicationPackage.fromFile(testApp); + MockRoot root = new MockRoot(); + new ConfigServerContainerModelBuilder(new TestOptions()).build(new DeployState.Builder().applicationPackage(app).build(), null, root, XML.getChild(XML.getDocument(app.getServices()).getDocumentElement(), "jdisc")); + root.freezeModelTopology(); + ElkConfig config = root.getConfig(ElkConfig.class, "configserver/configserver"); + assertThat(config.elasticsearch().size(), is(1)); + assertThat(config.elasticsearch(0).host(), is("foo")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java new file mode 100644 index 00000000000..110b4065c8f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespaclient.config.FeederConfig; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.11 + */ +public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBase { + + private Map<String, Handler<?>> getHandlers(String clusterName) { + ContainerCluster cluster = (ContainerCluster) root.getChildren().get(clusterName); + Map<String, Handler<?>> handlerMap = new HashMap<>(); + Collection<Handler<?>> handlers = cluster.getHandlers(); + for (Handler<?> handler : handlers) { + assertThat(handlerMap.containsKey(handler.getComponentId().toString()), is(false)); //die on overwrites + handlerMap.put(handler.getComponentId().toString(), handler); + } + return handlerMap; + } + + @Test + public void document_api_config_is_added_to_container_cluster() throws Exception { + Element elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + " <document-api>", + " <abortondocumenterror>false</abortondocumenterror>", + " <maxpendingdocs>4321</maxpendingdocs>", + " <retrydelay>12.34</retrydelay>", + " <route>non-default</route>", + " </document-api>", + nodesXml, + "</jdisc>"); + createModel(root, elem); + ContainerCluster cluster = (ContainerCluster)root.getProducer("cluster1"); + FeederConfig.Builder builder = new FeederConfig.Builder(); + cluster.getDocumentApi().getConfig(builder); + FeederConfig config = new FeederConfig(builder); + assertThat(config.abortondocumenterror(), is(false)); + assertThat(config.maxpendingdocs(), is(4321)); + assertThat(config.retrydelay(), is(12.34)); + assertThat(config.route(), is("non-default")); + } + + @Test + public void custom_bindings_are_allowed() throws Exception { + Element elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + " <document-api>", + " <binding>https://*/document-api/</binding>", + " <binding>missing-trailing-slash</binding>", + " </document-api>", + nodesXml, + "</jdisc>"); + createModel(root, elem); + + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandler", "feed"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerGet", "get"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus"); + verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit"); + verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"); + } + + private void verifyCustomBindings(String id, String bindingSuffix) { + Handler<?> handler = getHandlers("cluster1").get(id); + + assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix + "/")); + assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix + "/")); + + assertThat(handler.getServerBindings().size(), is(4)); + } + + @Test + public void requireThatHandlersAreSetup() throws Exception { + Element elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + " <document-api />", + nodesXml, + "</jdisc>"); + createModel(root, elem); + + Map<String, Handler<?>> handlerMap = getHandlers("cluster1"); + + assertThat(handlerMap.get("com.yahoo.container.config.StatisticsRequestHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.container.handler.VipStatusHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler"), not(nullValue())); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("http://*/feed"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("https://*/feed"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("http://*/feed/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("https://*/feed/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getComponentId().toString(), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("http://*/document"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("https://*/document"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("http://*/document/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("https://*/document/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("http://*/get"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("https://*/get"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("http://*/get/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("https://*/get/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("http://*/feedstatus"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("https://*/feedstatus"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("http://*/feedstatus/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("https://*/feedstatus/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("http://*/visit"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("https://*/visit"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("http://*/visit/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("https://*/visit/"), is(true)); + assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().size(), equalTo(4)); + + assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().contains("http://*/search/*"), is(true)); + assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().contains("https://*/search/*"), is(true)); + assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().size(), equalTo(2)); + + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(4)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java new file mode 100644 index 00000000000..3ea0a9732d6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -0,0 +1,573 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.config.StatisticsRequestHandler; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.servlet.ServletConfigConfig; +import com.yahoo.container.handler.VipStatusHandler; +import com.yahoo.container.handler.observability.ApplicationStatusHandler; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.usability.BindingsOverviewHandler; +import com.yahoo.jdisc.http.ServletPathsConfig; +import com.yahoo.prelude.cluster.QrMonitorConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.HttpFilter; +import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; + +import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + * @since 5.1.9 + */ +public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void default_port_is_4080() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc version='1.0'>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + AbstractService container = (AbstractService)root.getProducer("jdisc/container.0"); + assertThat(container.getRelativePort(0), is(Defaults.getDefaults().vespaWebServicePort())); + } + + @Test + public void http_server_port_is_configurable_and_does_not_affect_other_ports() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc version='1.0'>", + " <http>", + " <server port='9000' id='foo' />", + " </http>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + AbstractService container = (AbstractService)root.getProducer("jdisc/container.0"); + assertThat(container.getRelativePort(0), is(9000)); + assertThat(container.getRelativePort(1), is(not(9001))); + } + + @Test + public void fail_if_http_port_is_not_4080_in_hosted_vespa() throws Exception { + String servicesXml = + "<services>" + + "<admin version='3.0'>" + + " <nodes count='1'/>" + + "</admin>" + + "<jdisc version='1.0'>" + + " <http>" + + " <server port='9000' id='foo' />" + + " </http>" + + nodesXml + + "</jdisc>" + + "</services>"; + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + // Need to create VespaModel to make deploy properties have effect + final MyLogger logger = new MyLogger(); + new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .applicationPackage(applicationPackage) + .deployLogger(logger) + .properties(new DeployProperties.Builder() + .hostedVespa(true) + .build()) + .build()); + assertFalse(logger.msgs.isEmpty()); + assertThat(logger.msgs.get(0).getSecond(), containsString(String.format("You cannot set port to anything else than %d", Container.BASEPORT))); + } + + private class MyLogger implements DeployLogger { + List<Pair<Level, String>> msgs = new ArrayList<>(); + @Override + public void log(Level level, String message) { + msgs.add(new Pair<>(level, message)); + } + } + + @Test + public void one_cluster_with_explicit_port_and_one_without_is_ok() throws Exception { + Element cluster1Elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0' />"); + Element cluster2Elem = DomBuilderTest.parse( + "<jdisc id='cluster2' version='1.0'>", + " <http>", + " <server port='8000' id='foo' />", + " </http>", + "</jdisc>"); + createModel(root, cluster1Elem, cluster2Elem); + } + + @Test + public void two_clusters_without_explicit_port_throws_exception() throws SAXException, IOException { + Element cluster1Elem = DomBuilderTest.parse( + "<jdisc id='cluster1' version='1.0'>", + nodesXml, + "</jdisc>" ); + Element cluster2Elem = DomBuilderTest.parse( + "<jdisc id='cluster2' version='1.0'>", + nodesXml, + "</jdisc>" ); + try { + createModel(root, cluster1Elem, cluster2Elem); + fail("Expected exception"); + } catch (RuntimeException e) { + assertThat(e.getMessage(), containsString("cannot reserve port")); + } + } + + @Test + public void verify_bindings_for_builtin_handlers() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' />" + ); + createModel(root, clusterElem); + JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); + + JdiscBindingsConfig.Handlers defaultRootHandler = config.handlers(BindingsOverviewHandler.class.getName()); + assertThat(defaultRootHandler.serverBindings(), contains("*://*/")); + + JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); + assertThat(applicationStatusHandler.serverBindings(), + contains("http://*/ApplicationStatus", "https://*/ApplicationStatus")); + + JdiscBindingsConfig.Handlers statisticsRequestHandler = config.handlers(StatisticsRequestHandler.class.getName()); + assertTrue(statisticsRequestHandler.serverBindings(0).startsWith("http://*/statistics")); + assertTrue(statisticsRequestHandler.serverBindings(1).startsWith("https://*/statistics")); + + JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); + assertThat(fileRequestHandler.serverBindings(), + contains("http://*/status.html", "https://*/status.html")); + } + + @Test + public void default_root_handler_is_disabled_when_user_adds_a_handler_with_same_binding() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>" + + " <handler id='userRootHandler'>" + + " <binding>" + ContainerCluster.ROOT_HANDLER_BINDING + "</binding>" + + " </handler>" + + "</jdisc>"); + createModel(root, clusterElem); + + ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); + assertThat(userRootHandler, nullValue()); + } + + @Test + public void handler_bindings_are_included_in_discBindings_config() throws Exception { + createClusterWithJDiscHandler(); + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString("{discHandler}")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"clientBinding\"")); + } + + @Test + public void handlers_are_included_in_components_config() throws Exception { + createClusterWithJDiscHandler(); + assertThat(componentsConfig().toString(), containsString(".id \"discHandler\"")); + } + + private void createClusterWithJDiscHandler() throws SAXException, IOException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <handler id='discHandler'>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", + " <clientBinding>clientBinding</clientBinding>", + " </handler>", + "</jdisc>"); + + createModel(root, clusterElem); + } + + @Test + public void servlets_are_included_in_ServletPathConfig() throws Exception { + createClusterWithServlet(); + ServletPathsConfig servletPathsConfig = root.getConfig(ServletPathsConfig.class, "default"); + assertThat(servletPathsConfig.servlets().values().iterator().next().path(), is("p/a/t/h")); + } + + @Test + public void servletconfig_is_produced() throws Exception { + createClusterWithServlet(); + + String configId = getContainerCluster("default").getServletMap(). + values().iterator().next().getConfigId(); + + ServletConfigConfig servletConfig = root.getConfig(ServletConfigConfig.class, configId); + + assertThat(servletConfig.map().get("myKey"), is("myValue")); + } + + private void createClusterWithServlet() throws SAXException, IOException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <servlet id='myServlet' class='myClass' bundle='myBundle'>", + " <path>p/a/t/h</path>", + " <servlet-config>", + " <myKey>myValue</myKey>", + " </servlet-config>", + " </servlet>", + "</jdisc>"); + + createModel(root, clusterElem); + } + + + @Test + public void processing_handler_bindings_can_be_overridden() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <processing>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", + " </processing>", + "</jdisc>"); + + createModel(root, clusterElem); + + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, not(containsString("/processing/*"))); + } + + @Test + public void clientProvider_bindings_are_included_in_discBindings_config() throws Exception { + createModelWithClientProvider(); + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString("{discClient}")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"serverBinding\"")); + } + + @Test + public void clientProviders_are_included_in_components_config() throws Exception { + createModelWithClientProvider(); + assertThat(componentsConfig().toString(), containsString(".id \"discClient\"")); + } + + private void createModelWithClientProvider() throws SAXException, IOException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>" + + " <client id='discClient'>" + + " <binding>binding0</binding>" + + " <binding>binding1</binding>" + + " <serverBinding>serverBinding</serverBinding>" + + " </client>" + + "</jdisc>" ); + + createModel(root, clusterElem); + } + + @Test + public void serverProviders_are_included_in_components_config() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>" + + " <server id='discServer' />" + + "</jdisc>" ); + + createModel(root, clusterElem); + + String componentsConfig = componentsConfig().toString(); + assertThat(componentsConfig, containsString(".id \"discServer\"")); + } + + private String getChainsConfig(String configId) { + return root.getConfig(ChainsConfig.class, configId).toString(); + } + + @Test + public void searchHandler_gets_only_search_chains_in_chains_config() throws Exception { + createClusterWithProcessingAndSearchChains(); + String searchHandlerConfigId = "default/component/com.yahoo.search.handler.SearchHandler"; + String chainsConfig = getChainsConfig(searchHandlerConfigId); + assertThat(chainsConfig, containsLineWithPattern(".*\\.id \"testSearcher@default\"$")); + assertThat(chainsConfig, not(containsLineWithPattern(".*\\.id \"testProcessor@default\"$"))); + } + + @Test + public void processingHandler_gets_only_processing_chains_in_chains_config() throws Exception { + createClusterWithProcessingAndSearchChains(); + String processingHandlerConfigId = "default/component/com.yahoo.processing.handler.ProcessingHandler"; + String chainsConfig = getChainsConfig(processingHandlerConfigId); + assertThat(chainsConfig, containsLineWithPattern(".*\\.id \"testProcessor@default\"$")); + assertThat(chainsConfig, not(containsLineWithPattern(".*\\.id \"testSearcher@default\"$"))); + } + + private void createClusterWithProcessingAndSearchChains() throws SAXException, IOException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>" + + " <search>" + + " <chain id='default'>" + + " <searcher id='testSearcher' />" + + " </chain>" + + " </search>" + + " <processing>" + + " <chain id='default'>" + + " <processor id='testProcessor'/>" + + " </chain>" + + " </processing>" + + nodesXml + + " </jdisc>"); + + createModel(root, clusterElem); + } + + @Test + public void user_config_can_be_overridden_on_node() throws Exception { + Element containerElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <config name=\"prelude.cluster.qr-monitor\">" + + " <requesttimeout>111</requesttimeout>", + " </config> " + + " <nodes>", + " <node hostalias='host1' />", + " <node hostalias='host2'>", + " <config name=\"prelude.cluster.qr-monitor\">", + " <requesttimeout>222</requesttimeout>", + " </config> ", + " </node>", + " </nodes>", + "</jdisc>"); + + root = ContentClusterUtils.createMockRoot(new String[]{"host1", "host2"}); + createModel(root, containerElem); + ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default"); + assertThat(cluster.getContainers().size(), is(2)); + assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.0").requesttimeout(), 111); + assertEquals(root.getConfig(QrMonitorConfig.class, "default/container.1").requesttimeout(), 222); + } + + @Test + public void http_section_can_be_set_up() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <http>", + " <chain id='filterChain2'>", + " <filter id='filter' />", + " </chain>", + " </http>", + "</jdisc>"); + + createModel(root, clusterElem); + + root.getChildren(); + } + + @Test + public void legacy_yca_filter_and_its_config_provider_are_included_in_components_config() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <filter id='YcaFilter' /> ", + "</jdisc>"); + + createModel(root, clusterElem); + assertThat(componentsConfig().toString(), containsString(".id \"YcaFilter\"")); + + String providerId = HttpFilter.configProviderId(ComponentId.fromString("YcaFilter")).stringValue(); + assertThat(componentsConfig().toString(), containsString(".id \"" + providerId + "\"")); + } + + @Test + public void nested_components_are_injected_to_handlers() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <handler id='myHandler'>", + " <component id='injected' />", + " </handler>", + " <client id='myClient'>", // remember, a client is also a request handler + " <component id='injected' />", + " </client>", + "</jdisc>"); + + createModel(root, clusterElem); + Component<?,?> handler = getContainerComponent("default", "myHandler"); + assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); + + Component<?,?> client = getContainerComponent("default", "myClient"); + assertThat(client.getInjectedComponentIds(), hasItem("injected@myClient")); + } + + @Test + public void component_includes_are_added() { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/application/include_dirs"); + VespaModel model = creator.create(true); + ContainerCluster cluster = model.getContainerClusters().get("default"); + Map<ComponentId, Component<?, ?>> componentsMap = cluster.getComponentsMap(); + Component<?,?> example = componentsMap.get( + ComponentId.fromString("test.Exampledocproc")); + assertThat(example.getComponentId().getName(), is("test.Exampledocproc")); + } + + @Test + public void affinity_is_set() throws IOException, SAXException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <http>", + " <server port='" + Defaults.getDefaults().vespaWebServicePort() + "' id='main' />", + " </http>", + " <nodes cpu-socket-affinity='true'>", + " <node hostalias='node1' />", + " <node hostalias='node2'> <server-port id='main' port='5080'/> </node>", + " <node hostalias='node3'> <server-port id='main' port='6080'/> </node>", + " <node hostalias='node4'> <server-port id='main' port='7080'/> </node>", + " </nodes>" + + "</jdisc>"); + createModel(root, clusterElem); + assertTrue(getContainerCluster("default").getContainers().get(0).getAffinity().isPresent()); + assertTrue(getContainerCluster("default").getContainers().get(1).getAffinity().isPresent()); + assertTrue(getContainerCluster("default").getContainers().get(2).getAffinity().isPresent()); + assertTrue(getContainerCluster("default").getContainers().get(3).getAffinity().isPresent()); + + assertThat(getContainerCluster("default").getContainers().get(0).getAffinity().get().cpuSocket(), is(0)); + assertThat(getContainerCluster("default").getContainers().get(1).getAffinity().get().cpuSocket(), is(1)); + assertThat(getContainerCluster("default").getContainers().get(2).getAffinity().get().cpuSocket(), is(2)); + assertThat(getContainerCluster("default").getContainers().get(3).getAffinity().get().cpuSocket(), is(3)); + } + + @Test + public void singlenode_servicespec_is_used_with_hosts_xml() throws IOException, SAXException { + String servicesXml = "<jdisc id='default' version='1.0' />"; + String hostsXml = "<hosts>\n" + + " <host name=\"test1.yahoo.com\">\n" + + " <alias>node1</alias>\n" + + " </host>\n" + + "</hosts>"; + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withHosts(hostsXml) + .withServices(servicesXml) + .build(); + VespaModel model = new VespaModel(applicationPackage); + assertThat(model.getHostSystem().getHosts().size(), is(1)); + } + + @Test + public void http_aliases_are_stored_on_cluster_and_on_service_properties() throws SAXException, IOException { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <aliases>", + " <service-alias>service1</service-alias>", + " <service-alias>service2</service-alias>", + " <endpoint-alias>foo1.bar1.com</endpoint-alias>", + " <endpoint-alias>foo2.bar2.com</endpoint-alias>", + " </aliases>", + " <nodes>", + " <node hostalias='host1' />", + " </nodes>", + "</jdisc>"); + + createModel(root, clusterElem); + assertEquals(getContainerCluster("default").serviceAliases().get(0), "service1"); + assertEquals(getContainerCluster("default").endpointAliases().get(0), "foo1.bar1.com"); + assertEquals(getContainerCluster("default").serviceAliases().get(1), "service2"); + assertEquals(getContainerCluster("default").endpointAliases().get(1), "foo2.bar2.com"); + + assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("servicealiases"), "service1,service2"); + assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("endpointaliases"), "foo1.bar1.com,foo2.bar2.com"); + + } + + @Test + public void singlenode_servicespec_is_used_with_hosted_vespa() throws IOException, SAXException { + String servicesXml = "<jdisc id='default' version='1.0' />"; + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com")) + .applicationPackage(applicationPackage) + .properties(new DeployProperties.Builder() + .multitenant(true) + .hostedVespa(true) + .build()) + .build()); + assertEquals(1, model.getHostSystem().getHosts().size()); + } + + @Test(expected = IllegalArgumentException.class) + public void renderers_named_JsonRenderer_are_not_allowed() throws IOException, SAXException { + createModel(root, generateContainerElementWithRenderer("JsonRenderer")); + } + + @Test(expected = IllegalArgumentException.class) + public void renderers_named_DefaultRenderer_are_not_allowed() throws IOException, SAXException { + createModel(root, generateContainerElementWithRenderer("DefaultRenderer")); + } + + @Test + public void renderers_named_something_else_are_allowed() throws IOException, SAXException { + createModel(root, generateContainerElementWithRenderer("my-little-renderer")); + } + + @Test + public void vip_status_handler_uses_file_for_hosted_vespa() throws Exception { + String servicesXml = "<services>" + + "<jdisc version='1.0'>" + + nodesXml + + "</jdisc>" + + "</services>"; + + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() + .applicationPackage(applicationPackage) + .properties(new DeployProperties.Builder() + .hostedVespa(true) + .build()) + .build()); + + AbstractConfigProducerRoot modelRoot = model.getRoot(); + VipStatusConfig vipStatusConfig = modelRoot.getConfig(VipStatusConfig.class, "jdisc/component/status.html-status-handler"); + assertTrue(vipStatusConfig.accessdisk()); + assertEquals(ContainerModelBuilder.HOSTED_VESPA_STATUS_FILE, vipStatusConfig.statusfile()); + } + + private Element generateContainerElementWithRenderer(String rendererId) { + return DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search>", + String.format(" <renderer id='%s'/>", rendererId), + " </search>", + "</jdisc>"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java new file mode 100644 index 00000000000..b5410f332f6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import org.junit.Before; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Collections; + +/** + * Utility functions for testing the ContainerModelBuilder + * + * @author gjoranv + * @since 5.5 + */ +public abstract class ContainerModelBuilderTestBase { + + public static final String nodesXml = + " <nodes>" + + " <node hostalias='mockhost' />" + + " </nodes>"; + protected MockRoot root; + + public static void createModel(MockRoot root, Element... containerElems) throws SAXException, IOException { + for (Element containerElem : containerElems) { + ContainerModel model = new ContainerModelBuilder(false, ContainerModelBuilder.Networking.enable).build(DeployState.createTestState(), null, root, containerElem); + ContainerCluster cluster = model.getCluster(); + generateDefaultSearchChains(cluster); + } + root.freezeModelTopology(); + } + + private static void generateDefaultSearchChains(ContainerCluster cluster) { + ContainerSearch search = cluster.getSearch(); + if (search != null) + search.initializeSearchChains(Collections.<String, AbstractSearchCluster>emptyMap()); + } + + @Before + public void prepareTest() throws Exception { + root = new MockRoot("root"); + } + + protected ComponentsConfig componentsConfig() { + return root.getConfig(ComponentsConfig.class, "default"); + } + + protected ComponentsConfig.Components getComponent(ComponentsConfig componentsConfig, String id) { + for (ComponentsConfig.Components component : componentsConfig.components()) { + if (component.id().equals(id)) + return component; + } + return null; + } + + public ContainerCluster getContainerCluster(String clusterId) { + return (ContainerCluster) root.getChildren().get(clusterId); + } + + public Component<?, ?> getContainerComponent(String clusterId, String componentId) { + return getContainerCluster(clusterId).getComponentsMap().get( + ComponentId.fromString(componentId)); + } + + // TODO: will not work with multiple instances of the same class + public Component<?, ?> getContainerComponentNested(String clusterId, String componentId) { + ComponentId id = ComponentId.fromString(componentId); + for (Component<?,?> component : getContainerCluster(clusterId).getAllComponents()) + if (id.equals(component.getComponentId())) + return component; + return null; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java new file mode 100644 index 00000000000..f38e831cdbb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.docproc.DocprocConfig; +import com.yahoo.config.docproc.SchemamappingConfig; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.BundlesConfig; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.Assert.*; + +/** + * @author einarmr + * @author gjoranv + * @since 5.1.9 + */ +public class DocprocBuilderTest extends DomBuilderTest { + + private ContainerCluster cluster; + private DocumentmanagerConfig documentmanagerConfig; + private ContainerMbusConfig containerMbusConfig; + private ComponentsConfig componentsConfig; + private ChainsConfig chainsConfig; + private BundlesConfig bundlesConfig; + private SchemamappingConfig schemamappingConfig; + private DocprocConfig docprocConfig; + private QrStartConfig qrStartConfig; + + @Before + public void setupCluster() { + ContainerModel model = new ContainerModelBuilder(false, Networking.disable).build(DeployState.createTestState(), null, root, servicesXml()); + cluster = model.getCluster(); + cluster.getDocproc().getChains().addServersAndClientsForChains(); + root.freezeModelTopology(); + + containerMbusConfig = root.getConfig(ContainerMbusConfig.class, cluster.getContainers().get(0).getConfigId()); + componentsConfig = root.getConfig(ComponentsConfig.class, cluster.getConfigId()); + chainsConfig = root.getConfig(ChainsConfig.class, + cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + + documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); + bundlesConfig = root.getConfig(BundlesConfig.class, cluster.getConfigId()); + schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); + qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); + docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); + } + + private Element servicesXml() { + return parse( + "<jdisc id='banan' version='1.0'>", + " <nodes>", + " <node hostalias='mockhost' baseport='1500' />", + " </nodes>", + " <document-processing compressdocuments='true' preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", + " <documentprocessor id='docproc1' class='com.yahoo.Docproc1' bundle='docproc1bundle'/>", + " <chain id='chein'>", + " <documentprocessor id='docproc2'/>", + " </chain>", + " </document-processing>", + "</jdisc>"); + } + + // TODO: re-enable assertions when the appropriate attributes are handled by the builder + @Test + public void testDocprocCluster() { + assertThat(cluster.getName(), is("banan")); + assertThat(cluster.getDocproc().isCompressDocuments(), is(true)); + //assertThat(cluster.getContainerDocproc().isPreferLocalNode(), is(true)); + //assertThat(cluster.getContainerDocproc().getNumNodesPerClient(), is(2)); + List<Container> services = cluster.getContainers(); + assertThat(services.size(), is(1)); + Container service = services.get(0); + assertThat(service, notNullValue()); + + Map<String, DocprocChain> chains = new HashMap<>(); + for (DocprocChain chain : cluster.getDocprocChains().allChains().allComponents()) { + chains.put(chain.getId().stringValue(), chain); + } + assertThat(chains.size(), is(1)); + + DocprocChain chain = chains.get("chein"); + assertThat(chain.getId().stringValue(), is("chein")); + assertThat(chain.getInnerComponents().size(), is(1)); + DocumentProcessor processor = chain.getInnerComponents().iterator().next(); + assertThat(processor.getComponentId().stringValue(), is("docproc2")); + } + + @Test + public void testDocumentManagerConfig() { + assertThat(documentmanagerConfig.enablecompression(), is(true)); + } + + @Test + public void testDocprocConfig() { + assertThat(docprocConfig.maxqueuetimems(), is(200000)); + + } + + @Test + public void testContainerMbusConfig() { + assertThat(containerMbusConfig.enabled(), is(true)); + assertTrue(containerMbusConfig.port() >= HostResource.BASE_PORT); + assertThat(containerMbusConfig.maxpendingcount(), is(300)); + assertThat(containerMbusConfig.maxpendingsize(), is(100)); + } + + @Test + public void testComponentsConfig() { + Map<String, ComponentsConfig.Components> components = new HashMap<>(); + for (ComponentsConfig.Components component : componentsConfig.components()) { + System.err.println(component.id()); + components.put(component.id(), component); + } + + ComponentsConfig.Components docprocHandler = components.get("com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + assertThat(docprocHandler.id(), is("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); + assertThat(docprocHandler.configId(), is("banan/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler")); + assertThat(docprocHandler.classId(), is("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); + assertThat(docprocHandler.bundle(), is("container-search-and-docproc")); + + ComponentsConfig.Components docproc1 = components.get("docproc1"); + assertThat(docproc1.id(), is("docproc1")); + assertThat(docproc1.configId(), is("banan/docprocchains/component/docproc1")); + assertThat(docproc1.classId(), is("com.yahoo.Docproc1")); + assertThat(docproc1.bundle(), is("docproc1bundle")); + + ComponentsConfig.Components docproc2 = components.get("docproc2@chein"); + assertThat(docproc2.id(), is("docproc2@chein")); + assertThat(docproc2.configId(), is("banan/docprocchains/chain/chein/component/docproc2")); + assertThat(docproc2.classId(), is("docproc2")); + assertThat(docproc2.bundle(), is("docproc2")); +/* + ComponentsConfig.Components health = components.get("com.yahoo.container.jdisc.state.StateHandler"); + assertThat(health.id(), is("com.yahoo.container.jdisc.state.StateHandler")); + assertThat(health.classId(), is("com.yahoo.container.jdisc.state.StateHandler")); + assertThat(health.bundle(), is("com.yahoo.container.jdisc.state.StateHandler")); +*/ + ComponentsConfig.Components sourceClient = components.get("source@MbusClient"); + assertNotNull(sourceClient); + assertThat(sourceClient.classId(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + assertThat(sourceClient.bundle(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + + ComponentsConfig.Components intermediateClient = components.get("chain.chein@MbusClient"); + assertNotNull(intermediateClient); + assertThat(intermediateClient.classId(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + assertThat(intermediateClient.bundle(), is("com.yahoo.container.jdisc.messagebus.MbusClientProvider")); + } + + @Test + public void testChainsConfig() { + Map<String, ChainsConfig.Components> components = new HashMap<>(); + for (ChainsConfig.Components component : chainsConfig.components()) { + components.put(component.id(), component); + } + + assertThat(components.size(), is(2)); + + ChainsConfig.Components docproc1 = components.get("docproc1"); + assertThat(docproc1.id(), is("docproc1")); + assertThat(docproc1.dependencies().provides().size(), is(0)); + assertThat(docproc1.dependencies().before().size(), is(0)); + assertThat(docproc1.dependencies().after().size(), is(0)); + + ChainsConfig.Components docproc2 = components.get("docproc2@chein"); + assertThat(docproc2.id(), is("docproc2@chein")); + assertThat(docproc2.dependencies().provides().size(), is(0)); + assertThat(docproc2.dependencies().before().size(), is(0)); + assertThat(docproc2.dependencies().after().size(), is(0)); + + Map<String, ChainsConfig.Chains> chainsMap = new HashMap<>(); + for (ChainsConfig.Chains chain : chainsConfig.chains()) { + chainsMap.put(chain.id(), chain); + } + + assertThat(chainsMap.size(), is(1)); + assertThat(chainsMap.get("chein").id(), is("chein")); + assertThat(chainsMap.get("chein").components().size(), is(1)); + assertThat(chainsMap.get("chein").components(0), is("docproc2@chein")); + assertThat(chainsMap.get("chein").inherits().size(), is(0)); + assertThat(chainsMap.get("chein").excludes().size(), is(0)); + assertThat(chainsMap.get("chein").phases().size(), is(0)); + } + + @Test + public void testBundlesConfig() { + assertThat(bundlesConfig.bundle().size(), is(0)); + } + + @Test + public void testSchemaMappingConfig() { + assertThat(schemamappingConfig.fieldmapping().size(), is(0)); + } + + @Test + public void testQrStartConfig() { + QrStartConfig.Jvm jvm = qrStartConfig.jvm(); + assertThat(jvm.server(), is(true)); + assertThat(jvm.verbosegc(), is(false)); + assertThat(jvm.gcopts(), is("")); + assertThat(jvm.heapsize(), is(1536)); + assertThat(jvm.stacksize(), is(512)); + assertThat(qrStartConfig.ulimitv(), is("")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java new file mode 100644 index 00000000000..453b8d0ffa6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.jdisc.FilterBindingsProvider; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.http.JettyHttpServer; +import org.junit.Test; +import org.w3c.dom.Element; + +import java.util.List; + +import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; + +/** + * @author einarmr + * @since 5.15 + */ +public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void verify_that_overriding_connector_options_works() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>\n" + + " <http>\n" + + " <server id='bananarama' port='4321'>\n" + + " <config name='jdisc.http.connector'>\n" + + " <requestHeaderSize>300000</requestHeaderSize>\n" + + " <headerCacheSize>300000</headerCacheSize>\n" + + " </config>\n" + + " </server>\n" + + " </http>\n" + + nodesXml + + "</jdisc>\n" + ); + createModel(root, clusterElem); + ConnectorConfig.Builder connectorConfigBuilder = new ConnectorConfig.Builder(); + ConnectorConfig cfg = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/bananarama"); + assertThat(cfg.requestHeaderSize(), is(300000)); + assertThat(cfg.headerCacheSize(), is(300000)); + } + + @Test + public void verify_that_enabling_jetty_works() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>" + + nodesXml + + "</jdisc>" + ); + createModel(root, clusterElem); + assertJettyServerInConfig(); + } + + @Test + public void verify_that_enabling_jetty_works_for_custom_http_servers() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='foo' />", + " </http>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + assertJettyServerInConfig(); + } + + @Test + public void verifyThatJettyHttpServerHasFilterBindingsProvider() throws Exception { + final Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + + final ComponentsConfig.Components jettyHttpServerComponent = extractComponentByClassName( + containerComponentsConfig(), com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName()); + assertThat(jettyHttpServerComponent, is(not(nullValue()))); + + final ComponentsConfig.Components filterBindingsProviderComponent = extractComponentByClassName( + containerComponentsConfig(), FilterBindingsProvider.class.getName()); + assertThat(filterBindingsProviderComponent, is(not(nullValue()))); + + final ComponentsConfig.Components.Inject filterBindingsProviderInjection = extractInjectionById( + jettyHttpServerComponent, filterBindingsProviderComponent.id()); + assertThat(filterBindingsProviderInjection, is(not(nullValue()))); + } + + @Test + public void verifyThatJettyHttpServerHasFilterBindingsProviderForCustomHttpServers() throws Exception { + final Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='foo' />", + " </http>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + + final ComponentsConfig.Components jettyHttpServerComponent = extractComponentByClassName( + clusterComponentsConfig(), com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName()); + assertThat(jettyHttpServerComponent, is(not(nullValue()))); + + final ComponentsConfig.Components filterBindingsProviderComponent = extractComponentByClassName( + clusterComponentsConfig(), FilterBindingsProvider.class.getName()); + assertThat(filterBindingsProviderComponent, is(not(nullValue()))); + + final ComponentsConfig.Components.Inject filterBindingsProviderInjection = extractInjectionById( + jettyHttpServerComponent, filterBindingsProviderComponent.id()); + assertThat(filterBindingsProviderInjection, is(not(nullValue()))); + } + + @Test + public void verify_that_old_http_config_override_inside_server_tag_works() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0' jetty='true'>", + " <http>", + " <server port='9000' id='foo'>", + " <config name=\"container.jdisc.config.http-server\">", + " <tcpKeepAliveEnabled>true</tcpKeepAliveEnabled>", + " <tcpNoDelayEnabled>false</tcpNoDelayEnabled>", + " <tcpListenBacklogLength>2</tcpListenBacklogLength>", + " <idleConnectionTimeout>34.1</idleConnectionTimeout>", + " <soLinger>42.2</soLinger>", + " <sendBufferSize>1234</sendBufferSize>", + " <maxHeaderSize>4321</maxHeaderSize>", + " <ssl>", + " <enabled>true</enabled>", + " <keyStoreType>JKS</keyStoreType>", + " <keyStorePath>apple</keyStorePath>", + " <trustStorePath>grape</trustStorePath>", + " <keyDBKey>tomato</keyDBKey>", + " <algorithm>onion</algorithm>", + " <protocol>carrot</protocol>", + " </ssl>", + " </config>", + " </server>", + " </http>", + nodesXml, + "</jdisc>" ); + createModel(root, clusterElem); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); + List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class); + + assertThat(jettyServers.size(), is(1)); + + JettyHttpServer server = jettyServers.get(0); + assertThat(server.model.bundleInstantiationSpec.classId.toString(), + is(com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName())); + assertThat(server.model.bundleInstantiationSpec.bundle.toString(), is("jdisc_http_service")); + assertThat(server.getConnectorFactories().size(), is(1)); + + ConnectorConfig.Builder connectorConfigBuilder = new ConnectorConfig.Builder(); + server.getConnectorFactories().get(0).getConfig(connectorConfigBuilder); + ConnectorConfig connector = new ConnectorConfig(connectorConfigBuilder); + assertThat(connector.name(), equalTo("foo")); + assertThat(connector.tcpKeepAliveEnabled(), equalTo(true)); + assertThat(connector.tcpNoDelay(), equalTo(false)); + assertThat(connector.acceptQueueSize(), equalTo(2)); + assertThat(connector.idleTimeout(), equalTo(34.1)); + assertThat(connector.soLingerTime(), equalTo(42)); + assertThat(connector.outputBufferSize(), equalTo(1234)); + assertThat(connector.headerCacheSize(), equalTo(4321)); + assertThat(connector.ssl().enabled(), equalTo(true)); + assertThat(connector.ssl().keyStoreType(), equalTo(KeyStoreType.Enum.JKS)); + assertThat(connector.ssl().keyStorePath(), equalTo("apple")); + assertThat(connector.ssl().trustStorePath(), equalTo("grape")); + assertThat(connector.ssl().keyDbKey(), equalTo("tomato")); + assertThat(connector.ssl().sslKeyManagerFactoryAlgorithm(), equalTo("onion")); + assertThat(connector.ssl().protocol(), equalTo("carrot")); + + assertThat( + extractComponentByClassName( + clusterComponentsConfig(), + com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName()), + is(not(nullValue()))); + } + + private void assertJettyServerInConfig() { + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); + List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class); + + assertThat(jettyServers.size(), is(1)); + + JettyHttpServer server = jettyServers.get(0); + assertThat(server.model.bundleInstantiationSpec.classId.toString(), + is(com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName())); + assertThat(server.model.bundleInstantiationSpec.bundle.toString(), is("jdisc_http_service")); + assertThat(server.getConnectorFactories().size(), is(1)); + + assertThat( + extractComponentByClassName( + containerComponentsConfig(), + com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName()), + is(not(nullValue()))); + } + + private static ComponentsConfig.Components extractComponentByClassName( + final ComponentsConfig componentsConfig, final String className) { + for (final ComponentsConfig.Components component : componentsConfig.components()) { + if (className.equals(component.classId())) { + return component; + } + } + return null; + } + + private static ComponentsConfig.Components.Inject extractInjectionById( + final ComponentsConfig.Components component, final String id) { + for (final ComponentsConfig.Components.Inject injection : component.inject()) { + if (id.equals(injection.id())) { + return injection; + } + } + return null; + } + + private ComponentsConfig containerComponentsConfig() { + final ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); + return root.getConfig( + ComponentsConfig.class, + cluster.getContainers().get(0).getConfigId()); + } + + private ComponentsConfig clusterComponentsConfig() { + return componentsConfig(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java new file mode 100644 index 00000000000..269edf6c5ad --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -0,0 +1,194 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.test.Matchers.hasItemWithMethod; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.*; + +/** + * @author gjoranv + * @since 5.1.10 + */ +public class SearchBuilderTest extends ContainerModelBuilderTestBase { + + private ChainsConfig chainsConfig() { + return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); + } + + + @Test + public void search_handler_bindings_can_be_overridden() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", + " </search>", + nodesXml, + "</jdisc>"); + + createModel(root, clusterElem); + + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, not(containsString("/search/*"))); + } + + @Test + public void search_handler_bindings_can_be_disabled() throws Exception { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search>", + " <binding/>", + " </search>", + nodesXml, + "</jdisc>"); + + createModel(root, clusterElem); + + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, not(containsString("/search/*"))); + } + + // TODO: remove test when all containers are named 'container' + @Test + public void cluster_with_only_search_gets_qrserver_as_service_name() throws Exception { + createClusterWithOnlyDefaultChains(); + ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default"); + assertThat(cluster.getContainers().get(0).getServiceName(), is("qrserver")); + } + + @Test + public void empty_search_element_gives_default_chains() throws Exception { + createClusterWithOnlyDefaultChains(); + assertThat(chainsConfig().chains(), hasItemWithMethod("vespaPhases", "id")); + assertThat(chainsConfig().chains(), hasItemWithMethod("native", "id")); + assertThat(chainsConfig().chains(), hasItemWithMethod("vespa", "id")); + } + + private void createClusterWithOnlyDefaultChains() throws SAXException, IOException { + Element containerElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search/>", + " <nodes>", + " <node hostalias='mockhost' />", + " </nodes>", + "</jdisc>"); + + createModel(root, containerElem); + } + + @Test + public void manually_setting_up_search_handler_is_forbidden() throws IOException, SAXException { + try { + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <handler id='com.yahoo.search.handler.SearchHandler' />", + nodesXml, + " </jdisc>"); + + + createModel(root, clusterElem); + fail("Expected exception"); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Setting up com.yahoo.search.handler.SearchHandler manually is not supported")); + } + } + + @Test + public void cluster_is_connected_to_content_clusters() throws Exception { + String hosts = hostsXml(); + + String services = "" + + "<services>"+ + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost'/>" + + " </admin>" + + " <jdisc version='1.0' id='container'>"+ + " <search>" + + " <chain id='mychain' inherits='vespa'/>" + + " </search>" + + " <nodes>"+ + " <node hostalias=\"mockhost\" />"+ + " </nodes>"+ + " </jdisc>"+ + contentXml() + + "</services>"; + + VespaModel model = getVespaModelWithMusic(hosts, services); + + ContainerCluster cluster = model.getContainerClusters().get("container"); + assertFalse(cluster.getSearchChains().localProviders().isEmpty()); + } + + @Test + public void cluster_is_connected_to_search_clusters() throws Exception { + String hosts = hostsXml(); + + String services = "" + + "<services>"+ + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost'/>" + + " </admin>" + + " <jdisc version='1.0' id='container'>"+ + " <search>" + + " <chain id='mychain' inherits='vespa'/>" + + " </search>" + + " <nodes>"+ + " <node hostalias=\"mockhost\" />"+ + " </nodes>"+ + " </jdisc>"+ + contentXml() + + "</services>"; + + VespaModel model = getVespaModelWithMusic(hosts, services); + + ContainerCluster cluster = model.getContainerClusters().get("container"); + assertFalse(cluster.getSearchChains().localProviders().isEmpty()); + } + + + private VespaModel getVespaModelWithMusic(String hosts, String services) throws ParseException { + return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + } + + private String hostsXml() { + return "" + + "<hosts> " + + " <host name=\"node0\">" + + " <alias>mockhost</alias>" + + " </host>" + + "</hosts>"; + } + + private String contentXml() { + return " <content version=\"1.0\" id='content'>"+ + " <documents>\n" + + " <document type=\"music\" mode='index'/>\n" + + " </documents>\n" + + " <redundancy>3</redundancy>"+ + " <group>"+ + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>"+ + " </group>"+ + " </content>"; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java new file mode 100644 index 00000000000..bcb113687ec --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java @@ -0,0 +1,762 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.engines.ProtonEngine; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +public class ClusterTest extends ContentBaseTest { + + private final static String HOSTS = "<admin version='2.0'><adminserver hostalias='mockhost' /></admin>"; + + + ContentCluster parse(String xml) { + xml = HOSTS + xml; + TestRoot root = new TestDriver().buildModel(xml); + return root.getConfigModels(Content.class).get(0).getCluster(); + } + + @Test + public void testRedundancy() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + parse("" + + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <proton>" + + " <searchable-copies>3</searchable-copies>" + + " </proton>" + + " </engine>" + + " <redundancy reply-after=\"4\">5</redundancy>\n" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"3\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"4\"/>\"" + + " </group>" + + "</content>" + ).getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + assertEquals(4, config.initial_redundancy()); + assertEquals(5, config.redundancy()); + assertEquals(3, config.ready_copies()); + } + + @Test + public void testNoId() { + ContentCluster c = parse( + "<content version=\"1.0\">\n" + + " <redundancy>1</redundancy>\n" + + " <documents/>" + + " <redundancy reply-after=\"4\">5</redundancy>\n" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" + + " </group>" + + "</content>" + ); + + assertEquals("content", c.getName()); + } + + @Test + public void testRedundancyDefaults() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"1\"/>\"" + + " <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" + + " </group>" + + "</content>" + ).getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + assertEquals(2, config.initial_redundancy()); + assertEquals(3, config.redundancy()); + assertEquals(2, config.ready_copies()); + } + + @Test + public void testEndToEnd() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"configserver\" />\n" + + " <logserver hostalias=\"logserver\" />\n" + + " <slobroks>\n" + + " <slobrok hostalias=\"configserver\" />\n" + + " <slobrok hostalias=\"logserver\" />\n" + + " </slobroks>\n" + + " <cluster-controllers>\n" + + " <cluster-controller hostalias=\"configserver\"/>" + + " <cluster-controller hostalias=\"configserver2\"/>" + + " <cluster-controller hostalias=\"configserver3\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode=\"index\"/>\n" + + " <document type=\"type2\" mode=\"index\"/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " <tuning>" + + " <cluster-controller>\n" + + " <init-progress-time>34567</init-progress-time>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create(); + assertEquals(2, model.getContentClusters().get("bar").getDocumentDefinitions().size()); + ContainerCluster cluster = model.getAdmin().getClusterControllers(); + assertEquals(3, cluster.getContainers().size()); + } + + @Test + public void testEndToEndOneNode() throws Exception { + String services = + "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + " </jdisc>" + + " <content id='storage' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " <node distribution-key='1' hostalias='node1'/>" + + " </group>" + + " <tuning>" + + " <cluster-controller>" + + " <transition-time>0</transition-time>" + + " </cluster-controller>" + + " </tuning>" + + " <documents>" + + " <document mode='store-only' type='type1'/>" + + " </documents>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " </content>" + + " </services>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + VespaModel model = (new VespaModelCreatorWithMockPkg(null, services, sds)).create(); + assertEquals(1, model.getContentClusters().get("storage").getDocumentDefinitions().size()); + ContainerCluster cluster = model.getAdmin().getClusterControllers(); + assertEquals(1, cluster.getContainers().size()); + } + + @Test + public void testSearchTuning() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"node0\" />\n" + + " <cluster-controllers>\n" + + " <cluster-controller hostalias=\"node0\"/>" + + " </cluster-controllers>\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode='index'/>\n" + + " <document type=\"type2\" mode='index'/>\n" + + " </documents>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0'/>" + + " </group>" + + " <tuning>\n" + + " <cluster-controller>" + + " <init-progress-time>34567</init-progress-time>" + + " </cluster-controller>" + + " </tuning>" + + " </content>" + + "\n" + + "</services>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + + assertTrue(model.getContentClusters().get("bar").getPersistence() instanceof ProtonEngine.Factory); + + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + model.getConfig(builder, "bar/distributor/0"); + StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); + assertEquals(false, config.inlinebucketsplitting()); + } + + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + model.getConfig(builder, "bar/storage/0"); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(false, config.enable_multibit_split_optimalization()); + } + } + + @Test + public void testRedundancyRequired() throws Exception { + String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services>\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"node0\" />\n" + + " </admin>\n" + + " <content version='1.0' id='bar'>" + + " <documents>" + + " <document type=\"type1\" mode='index'/>\n" + + " </documents>\n" + + " <group>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </group>\n" + + " </content>\n" + + "</services>\n"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + try{ + new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + assertTrue("Deploying without redundancy should fail", false); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("missing required element \"redundancy\"")); + } + } + + @Test + public void testRedundancyFinalLessThanInitial() { + try { + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <redundancy reply-after=\"4\">2</redundancy>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + "</content>" + ); + fail("no exception thrown"); + } catch (Exception e) { + } + } + + @Test + public void testReadyTooHigh() { + try { + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <engine>" + + " <proton>" + + " <searchable-copies>3</searchable-copies>" + + " </proton>" + + " </engine>" + + " <redundancy>2</redundancy>\n" + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + "</content>" + ); + fail("no exception thrown"); + } catch (Exception e) { + } + } + + FleetcontrollerConfig getFleetControllerConfig(String xml) { + ContentCluster cluster = parse(xml); + + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + return new FleetcontrollerConfig(builder); + } + + @Test + public void testFleetControllerOverride() + { + { + FleetcontrollerConfig config = getFleetControllerConfig( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + + assertEquals(0, config.min_storage_up_ratio(), 0.01); + assertEquals(0, config.min_distributor_up_ratio(), 0.01); + assertEquals(1, config.min_storage_up_count()); + assertEquals(1, config.min_distributors_up_count()); + } + + { + FleetcontrollerConfig config = getFleetControllerConfig( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"2\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"3\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"4\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"5\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + + assertNotSame(0, config.min_storage_up_ratio()); + } + } + + @Test + public void testImplicitDistributionBits() + { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + + { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(8, config.ideal_distribution_bits()); + } + + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + cluster.getConfig(builder); + StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); + assertEquals(8, config.minsplitcount()); + } + cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + + { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(8, config.ideal_distribution_bits()); + } + + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + cluster.getConfig(builder); + StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); + assertEquals(8, config.minsplitcount()); + } + } + + @Test + public void testExplicitDistributionBits() + { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " <tuning>\n" + + " <distribution type=\"strict\"/>\n" + + " </tuning>\n" + + "</content>" + ); + + { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(8, config.ideal_distribution_bits()); + } + + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + cluster.getConfig(builder); + StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); + assertEquals(8, config.minsplitcount()); + } + cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " <tuning>\n" + + " <distribution type=\"loose\"/>\n" + + " </tuning>\n" + + "</content>" + ); + + { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(8, config.ideal_distribution_bits()); + } + + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + cluster.getConfig(builder); + StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); + assertEquals(8, config.minsplitcount()); + } + } + + @Test + public void testGenerateSearchNodes() + { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + cluster.getStorageNodes().getChildren().get("0").getConfig(builder); + StorServerConfig config = new StorServerConfig(builder); + assertEquals("tcp/localhost:19106", config.persistence_provider().rpc().connectspec()); + } + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + cluster.getStorageNodes().getChildren().get("1").getConfig(builder); + StorServerConfig config = new StorServerConfig(builder); + assertEquals("tcp/localhost:19118", config.persistence_provider().rpc().connectspec()); + } + } + + @Test + public void testAlternativeNodeSyntax() + { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"test\">\n" + + " <documents/>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <nodes>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </nodes>\n" + + "</content>" + ); + + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + + cluster.getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + + assertEquals("invalid", config.group(0).name()); + assertEquals("invalid", config.group(0).index()); + assertEquals(2, config.group(0).nodes().size()); + } + + @Test + public void testReadyWhenInitialOne() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <redundancy>1</redundancy>\n" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>" + ).getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + assertEquals(1, config.initial_redundancy()); + assertEquals(1, config.redundancy()); + assertEquals(1, config.ready_copies()); + } + + public void testProvider(String tagName, StorServerConfig.Persistence_provider.Type.Enum type) { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <redundancy>3</redundancy>" + + " <engine>\n" + + " <" + tagName + "/>\n" + + " </engine>\n" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>" + ); + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + cluster.getStorageNodes().getChildren().get("0").getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + + assertEquals(type, config.persistence_provider().type()); + } + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getDistributorNodes().getConfig(builder); + cluster.getDistributorNodes().getChildren().get("0").getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + + assertEquals(type, config.persistence_provider().type()); + } + } + + @Test + public void testProviders() { + testProvider("proton", StorServerConfig.Persistence_provider.Type.RPC); + testProvider("rpc", StorServerConfig.Persistence_provider.Type.RPC); + testProvider("vds", StorServerConfig.Persistence_provider.Type.STORAGE); + testProvider("dummy", StorServerConfig.Persistence_provider.Type.DUMMY); + } + + @Test + public void testMetrics() { + MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder(); + + ContentCluster cluster = parse("<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</content>" + ); + cluster.getConfig(builder); + + MetricsmanagerConfig config = new MetricsmanagerConfig(builder); + + assertEquals(6, config.consumer().size()); + assertEquals("status", config.consumer(0).name()); + assertEquals("*", config.consumer(0).addedmetrics(0)); + assertEquals("partofsum", config.consumer(0).removedtags(0)); + + assertEquals("log", config.consumer(1).name()); + assertEquals("logdefault", config.consumer(1).tags().get(0)); + assertEquals("loadtype", config.consumer(1).removedtags(0)); + + assertEquals("yamas", config.consumer(2).name()); + assertEquals("yamasdefault", config.consumer(2).tags().get(0)); + assertEquals("loadtype", config.consumer(2).removedtags(0)); + + assertEquals("health", config.consumer(3).name()); + + assertEquals("statereporter", config.consumer(5).name()); + assertEquals("*", config.consumer(5).addedmetrics(0)); + assertEquals("thread", config.consumer(5).removedtags(0)); + assertEquals("disk", config.consumer(5).tags(0)); + + cluster.getStorageNodes().getConfig(builder); + config = new MetricsmanagerConfig(builder); + assertEquals(6, config.consumer().size()); + + assertEquals("fleetcontroller", config.consumer(4).name()); + assertEquals(9, config.consumer(4).addedmetrics().size()); + assertEquals("vds.filestor.*.allthreads.put.sum", config.consumer(4).addedmetrics(0)); + assertEquals("vds.filestor.*.allthreads.get.sum", config.consumer(4).addedmetrics(1)); + assertEquals("vds.filestor.*.allthreads.multi.sum", config.consumer(4).addedmetrics(2)); + assertEquals("vds.filestor.*.allthreads.update.sum", config.consumer(4).addedmetrics(3)); + assertEquals("vds.filestor.*.allthreads.remove.sum", config.consumer(4).addedmetrics(4)); + assertEquals("vds.filestor.*.allthreads.operations", config.consumer(4).addedmetrics(5)); + assertEquals("vds.datastored.alldisks.docs", config.consumer(4).addedmetrics(6)); + assertEquals("vds.datastored.alldisks.bytes", config.consumer(4).addedmetrics(7)); + assertEquals("vds.datastored.alldisks.buckets", config.consumer(4).addedmetrics(8)); + } + + public MetricsmanagerConfig.Consumer getConsumer(String consumer, MetricsmanagerConfig config) { + for (MetricsmanagerConfig.Consumer c : config.consumer()) { + if (c.name().equals(consumer)) { + return c; + } + } + + return null; + } + + @Test + public void testConfiguredMetrics() throws Exception { + String xml = "" + + "<services>" + + "<content version=\"1.0\" id=\"storage\">\n" + + " <redundancy>1</redundancy>\n" + + " <documents>" + + " <document type=\"type1\" mode='index'/>\n" + + " <document type=\"type2\" mode='index'/>\n" + + " </documents>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + + " </group>\n" + + "</content>" + + "<admin version=\"2.0\">" + + " <logserver hostalias=\"node0\"/>" + + " <adminserver hostalias=\"node0\"/>" + + " <metric-consumers>" + + " <consumer name=\"foobar\">" + + " <metric name=\"storage.foo.bar\"/>" + + " </consumer>" + + " <consumer name=\"log\">" + + " <metric name=\"extralogmetric\"/>" + + " <metric name=\"extralogmetric3\"/>" + + " </consumer>" + + " <consumer name=\"fleetcontroller\">" + + " <metric name=\"extraextra\"/>" + + " </consumer>" + + " </metric-consumers>" + + "</admin>" + + "</services>"; + + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + + { + MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder(); + model.getConfig(builder, "storage/storage/0"); + MetricsmanagerConfig config = new MetricsmanagerConfig(builder); + + assertEquals("[storage.foo.bar]", getConsumer("foobar", config).addedmetrics().toString()); + String expected = + "[extralogmetric\n" + + "extralogmetric3\n" + + "vds.filestor.alldisks.allthreads.put.sum\n" + + "vds.filestor.alldisks.allthreads.get.sum\n" + + "vds.filestor.alldisks.allthreads.remove.sum\n" + + "vds.filestor.alldisks.allthreads.update.sum\n" + + "vds.datastored.alldisks.docs\n" + + "vds.datastored.alldisks.bytes\n" + + "vds.filestor.alldisks.queuesize\n" + + "vds.filestor.alldisks.averagequeuewait.sum\n" + + "vds.visitor.cv_queuewaittime\n" + + "vds.visitor.allthreads.averagequeuewait\n" + + "vds.visitor.allthreads.averagevisitorlifetime\n" + + "vds.visitor.allthreads.created.sum]"; + String actual = getConsumer("log", config).addedmetrics().toString().replaceAll(", ", "\n"); + assertEquals(expected, actual); + assertEquals("[logdefault]", getConsumer("log", config).tags().toString()); + expected = + "[extraextra\n" + + "vds.filestor.*.allthreads.put.sum\n" + + "vds.filestor.*.allthreads.get.sum\n" + + "vds.filestor.*.allthreads.multi.sum\n" + + "vds.filestor.*.allthreads.update.sum\n" + + "vds.filestor.*.allthreads.remove.sum\n" + + "vds.filestor.*.allthreads.operations\n" + + "vds.datastored.alldisks.docs\n" + + "vds.datastored.alldisks.bytes\n" + + "vds.datastored.alldisks.buckets]"; + actual = getConsumer("fleetcontroller", config).addedmetrics().toString().replaceAll(", ", "\n"); + assertEquals(expected, actual); + } + + { + MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder(); + model.getConfig(builder, "storage/distributor/0"); + MetricsmanagerConfig config = new MetricsmanagerConfig(builder); + + assertEquals("[storage.foo.bar]", getConsumer("foobar", config).addedmetrics().toString()); + assertEquals("[extralogmetric, extralogmetric3, vds.distributor.docsstored, vds.distributor.bytesstored, vds.idealstate.delete_bucket.done_ok, vds.idealstate.merge_bucket.done_ok, vds.idealstate.split_bucket.done_ok, vds.idealstate.join_bucket.done_ok, vds.idealstate.buckets_rechecking]", getConsumer("log", config).addedmetrics().toString()); + assertEquals("[logdefault]", getConsumer("log", config).tags().toString()); + assertEquals("[extraextra]", getConsumer("fleetcontroller", config).addedmetrics().toString()); + } + } + + @Test + public void requireThatPreShutdownCommandIsSet() { + ContentCluster cluster = parse( + "<content version=\"1.0\" id=\"storage\">" + + " <documents/>" + + " <engine>" + + " <proton>" + + " <flush-on-shutdown>true</flush-on-shutdown>" + + " </proton>" + + " </engine>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>"); + assertThat(cluster.getSearch().getSearchNodes().size(), is(1)); + assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent()); + + cluster = parse( + "<content version=\"1.0\" id=\"storage\">" + + " <documents/>" + + " <engine>" + + " <proton>" + + " <flush-on-shutdown> \n " + + " true </flush-on-shutdown>" + + " </proton>" + + " </engine>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>"); + assertThat(cluster.getSearch().getSearchNodes().size(), is(1)); + assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java new file mode 100644 index 00000000000..038179af9cf --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +public class ContentBaseTest { + public static String getHosts() { + return "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts> " + + " <host name='foo'>" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java new file mode 100644 index 00000000000..d866fd225b3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import junit.framework.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; +import static junit.framework.TestCase.assertEquals; + +/** + * Unit tests for content search cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ContentSearchClusterTest { + + private static double EPSILON = 0.000001; + + private static ContentCluster createClusterWithOneDocumentType() throws Exception { + return createCluster(new ContentClusterBuilder().getXml()); + } + + private static ContentCluster createClusterWithTwoDocumentType() throws Exception { + List<String> docTypes = Arrays.asList("foo", "bar"); + return createCluster(new ContentClusterBuilder().docTypes(docTypes).getXml(), + ApplicationPackageUtils.generateSearchDefinitions(docTypes)); + } + + private static ProtonConfig getProtonConfig(ContentCluster cluster) { + ProtonConfig.Builder protonCfgBuilder = new ProtonConfig.Builder(); + cluster.getSearch().getConfig(protonCfgBuilder); + return new ProtonConfig(protonCfgBuilder); + } + + @Test + public void requireThatProtonInitializeThreadsIsSet() throws Exception { + assertEquals(2, getProtonConfig(createClusterWithOneDocumentType()).initialize().threads()); + assertEquals(3, getProtonConfig(createClusterWithTwoDocumentType()).initialize().threads()); + } + + @Test + public void requireThatProtonResourceLimitsCanBeSet() throws Exception { + String clusterXml = new ContentClusterBuilder().protonDiskLimit(0.88).protonMemoryLimit(0.77).getXml(); + ProtonConfig cfg = getProtonConfig(createCluster(clusterXml)); + assertEquals(0.88, cfg.writefilter().disklimit(), EPSILON); + assertEquals(0.77, cfg.writefilter().memorylimit(), EPSILON); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchTest.java new file mode 100644 index 00000000000..5d3a1105289 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchTest.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ContentSearchTest { + + @Test + public void requireThatAccessorsWork() { + ContentSearch search = new ContentSearch.Builder() + .setQueryTimeout(1.0) + .setVisibilityDelay(2.0) + .build(); + assertEquals(1.0, search.getQueryTimeout(), 1E-6); + assertEquals(2.0, search.getVisibilityDelay(), 1E-6); + } + + @Test + public void requireThatDefaultsAreNull() { + ContentSearch search = new ContentSearch.Builder().build(); + assertNull(search.getQueryTimeout()); + assertNull(search.getVisibilityDelay()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java new file mode 100644 index 00000000000..d886b0feee3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -0,0 +1,374 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.hamcrest.Matchers.*; +/** + * Test for content DistributorCluster. + */ +public class DistributorTest { + + ContentCluster parseCluster(String xml) { + try { + final List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies"); + MockRoot root = ContentClusterUtils.createMockRoot(searchDefs); + return ContentClusterUtils.createCluster(xml, root); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + DistributorCluster parse(String xml) { + return parseCluster(xml).getDistributorNodes(); + } + + @Test + public void testBasics() { + + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse("<content id=\"foofighters\"><documents/>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>\n"). + getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(true, config.is_distributor()); + assertEquals("foofighters", config.cluster_name()); + } + + @Test + public void testRevertDefaultOffForSearch() { + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(true, conf.enable_revert()); + } + { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(false, conf.enable_revert()); + } + } + + @Test + public void testSplitAndJoin() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <tuning>\n" + + " <bucket-splitting max-documents=\"2K\" max-size=\"25M\" minimum-bits=\"8\" />\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + + assertEquals(2048, conf.splitcount()); + assertEquals(1024, conf.joincount()); + assertEquals(26214400, conf.splitsize()); + assertEquals(13107200, conf.joinsize()); + assertEquals(8, conf.minsplitcount()); + assertEquals(true, conf.inlinebucketsplitting()); + } + + @Test + public void testThatGroupsAreCountedInWhenComputingSplitBits() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + ContentCluster cluster = parseCluster("<cluster id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <tuning>" + + " <distribution type=\"legacy\"/>" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>"); + cluster.getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + + assertEquals(1024, conf.splitcount()); + assertEquals(512, conf.joincount()); + assertEquals(33544432, conf.splitsize()); + assertEquals(16000000, conf.joinsize()); + assertEquals(8, conf.minsplitcount()); + assertEquals(true, conf.inlinebucketsplitting()); + + cluster = parseCluster("<cluster id=\"storage\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <tuning>" + + " <distribution type=\"legacy\"/>" + + " </tuning>\n" + + " <group>" + + " <distribution partitions=\"1|*\"/>" + + " <group name=\"a\" distribution-key=\"0\">" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + " <group name=\"b\" distribution-key=\"1\">" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>" + + " </group>" + + " </group>" + + "</cluster>"); + cluster.getConfig(builder); + + conf = new StorDistributormanagerConfig(builder); + + assertEquals(1024, conf.splitcount()); + assertEquals(512, conf.joincount()); + assertEquals(33544432, conf.splitsize()); + assertEquals(16000000, conf.joinsize()); + assertEquals(1, conf.minsplitcount()); + assertEquals(true, conf.inlinebucketsplitting()); + } + + @Test + public void testMaxMergesPerNode() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + DistributorCluster dcluster = parse("<content id=\"storage\">\n" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>"); + ((ContentCluster) dcluster.getParent()).getConfig(builder); + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(16, conf.maximum_nodes_per_merge()); + + builder = new StorDistributormanagerConfig.Builder(); + dcluster = parse("<content id=\"storage\">\n" + + " <documents/>" + + " <tuning>\n" + + " <merges max-nodes-per-merge=\"4\"/>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>"); + ((ContentCluster) dcluster.getParent()).getConfig(builder); + conf = new StorDistributormanagerConfig(builder); + assertEquals(4, conf.maximum_nodes_per_merge()); + } + + @Test + public void testGarbageCollectionSetExplicitly() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents garbage-collection=\"true\">\n" + + " <document type=\"music\"/>\n" + + " </documents>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(3600, conf.garbagecollection().interval()); + assertEquals("not ((music))", conf.garbagecollection().selectiontoremove()); + } + + @Test + public void testGarbageCollectionInterval() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents garbage-collection=\"true\" garbage-collection-interval=\"30\">\n" + + " <document type=\"music\"/>\n" + + " </documents>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(30, conf.garbagecollection().interval()); + } + + @Test + public void testGarbageCollectionOffByDefault() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents>\n" + + " <document type=\"music\"/>\n" + + " </documents>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(0, conf.garbagecollection().interval()); + assertEquals("", conf.garbagecollection().selectiontoremove()); + } + + @Test + public void testComplexGarbageCollectionSelectionForIndexedSearch() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"foo\">\n" + + " <documents garbage-collection=\"true\" selection=\"true\">" + + " <document type=\"music\" selection=\"music.year < now()\"/>\n" + + " <document type=\"movies\" selection=\"movies.year < now() - 1200\"/>\n" + + " </documents>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(3600, conf.garbagecollection().interval()); + assertEquals( + "not ((true) and ((music and (music.year < now())) or (movies and (movies.year < now() - 1200))))", + conf.garbagecollection().selectiontoremove()); + } + + @Test + public void testGarbageCollectionDisabledIfForced() { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse("<cluster id=\"foo\">\n" + + " <documents selection=\"true\" garbage-collection=\"false\" garbage-collection-interval=\"30\">\n" + + " <document type=\"music\" selection=\"music.year < now()\"/>\n" + + " <document type=\"movies\" selection=\"movies.year < now() - 1200\"/>\n" + + " </documents>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>").getConfig(builder); + + StorDistributormanagerConfig conf = new StorDistributormanagerConfig(builder); + assertEquals(0, conf.garbagecollection().interval()); + assertEquals("", conf.garbagecollection().selectiontoremove()); + } + + @Test + public void testPortOverride() { + StorCommunicationmanagerConfig.Builder builder = new StorCommunicationmanagerConfig.Builder(); + DistributorCluster cluster = + parse("<cluster id=\"storage\" distributor-base-port=\"14065\">" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>"); + + cluster.getChildren().get("0").getConfig(builder); + StorCommunicationmanagerConfig config = new StorCommunicationmanagerConfig(builder); + assertEquals(14066, config.rpcport()); + } + + private StorDistributormanagerConfig clusterXmlToConfig(String xml) { + StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); + parse(xml).getConfig(builder); + return new StorDistributormanagerConfig(builder); + } + + private static class DocDef { + public final String type; + public final String mode; + + private DocDef(String type, String mode) { + this.type = type; + this.mode = mode; + } + + public static DocDef storeOnly(String type) { + return new DocDef(type, "store-only"); + } + + public static DocDef index(String type) { + return new DocDef(type, "index"); + } + + public static DocDef streaming(String type) { + return new DocDef(type, "streaming"); + } + } + + private String generateXmlForDocDefs(DocDef... defs) { + return "<content id='storage'>\n" + + " <documents>\n" + + Arrays.stream(defs) + .map(def -> String.format(" <document type='%s' mode='%s'/>", def.type, def.mode)) + .collect(Collectors.joining("\n")) + + "\n </documents>\n" + + "</content>"; + } + + @Test + public void bucket_activation_disabled_if_no_documents_in_indexed_mode() { + StorDistributormanagerConfig config = clusterXmlToConfig( + generateXmlForDocDefs(DocDef.storeOnly("music"))); + assertThat(config.disable_bucket_activation(), is(true)); + } + + @Test + public void bucket_activation_enabled_with_single_indexed_document() { + StorDistributormanagerConfig config = clusterXmlToConfig( + generateXmlForDocDefs(DocDef.index("music"))); + assertThat(config.disable_bucket_activation(), is(false)); + } + + @Test + public void bucket_activation_enabled_with_multiple_indexed_documents() { + StorDistributormanagerConfig config = clusterXmlToConfig( + generateXmlForDocDefs(DocDef.index("music"), + DocDef.index("movies"))); + assertThat(config.disable_bucket_activation(), is(false)); + } + + @Test + public void bucket_activation_enabled_if_at_least_one_document_indexed() { + StorDistributormanagerConfig config = clusterXmlToConfig( + generateXmlForDocDefs(DocDef.storeOnly("music"), + DocDef.streaming("bunnies"), + DocDef.index("movies"))); + assertThat(config.disable_bucket_activation(), is(false)); + } + + @Test + public void bucket_activation_disabled_for_single_streaming_type() { + StorDistributormanagerConfig config = clusterXmlToConfig( + generateXmlForDocDefs(DocDef.streaming("music"))); + assertThat(config.disable_bucket_activation(), is(true)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java new file mode 100644 index 00000000000..204491b0724 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.junit.Test; +import org.w3c.dom.Document; + +import static org.junit.Assert.assertEquals; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 5/10/12 + * Time: 2:29 PM + * To change this template use File | Settings | File Templates. + */ +public class FleetControllerClusterTest { + ClusterControllerConfig parse(String xml) { + Document doc = XML.getDocument(xml); + return new ClusterControllerConfig.Builder("storage", new ModelElement(doc.getDocumentElement())).build(new MockRoot(), + new ModelElement(doc.getDocumentElement()).getXml()); + } + + @Test + public void testParameters() { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents/>" + + " <tuning>\n" + + " <bucket-splitting minimum-bits=\"7\" />" + + " <cluster-controller>\n" + + " <init-progress-time>13</init-progress-time>\n" + + " <transition-time>27</transition-time>\n" + + " <max-premature-crashes>4</max-premature-crashes>\n" + + " <stable-state-period>72</stable-state-period>\n" + + " <min-distributor-up-ratio>0.7</min-distributor-up-ratio>\n" + + " <min-storage-up-ratio>0.3</min-storage-up-ratio>\n" + + " </cluster-controller>\n" + + " </tuning>\n" + + "</cluster>"). + getConfig(builder); + + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(13 * 1000, config.init_progress_time()); + assertEquals(27 * 1000, config.storage_transition_time()); + assertEquals(4, config.max_premature_crashes()); + assertEquals(72 * 1000, config.stable_state_time_period()); + assertEquals(0.7, config.min_distributor_up_ratio(), 0.01); + assertEquals(0.3, config.min_storage_up_ratio(), 0.01); + assertEquals(7, config.ideal_distribution_bits()); + } + + @Test + public void testDurationParameters() { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + parse("<cluster id=\"storage\">\n" + + " <documents/>" + + " <tuning>\n" + + " <cluster-controller>\n" + + " <init-progress-time>13ms</init-progress-time>\n" + + " </cluster-controller>\n" + + " </tuning>\n" + + "</cluster>"). + getConfig(builder); + + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(13, config.init_progress_time()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java new file mode 100644 index 00000000000..ba408bfbddf --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.storagecluster.StorageCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @since 5.1.8 + */ +public class GenericConfigTest { + + private VespaModel model; + + private String servicesXml() { + return "" + + "<services version='1.0'>" + + " <config name='vespa.config.storage.stor-memfilepersistence'>" + + " <disk_full_factor>0.001</disk_full_factor> " + + " </config>" + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\" />" + + " <cluster-controllers>" + + " <cluster-controller hostalias='node0'/>" + + " </cluster-controllers>" + + " </admin>" + + " <content version='1.0' id='storage'>" + + " <documents>" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " </documents>" + + " <config name='config.juniperrc'>" + + " <length>1024</length>" + + " </config>" + + " <redundancy>1</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node0'/>" + + " </group>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " </content>" + + "</services>"; + } + + @Before + public void getVespaModel() throws IOException, SAXException, ParseException { + model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSearchDefinitions("type1"))).create(); + } + + @Test + public void config_override_on_root_is_visible_on_storage_cluster() throws Exception { + StorageCluster cluster = model.getContentClusters().get("storage").getStorageNodes(); + + StorMemfilepersistenceConfig config = model.getConfig(StorMemfilepersistenceConfig.class, cluster.getConfigId()); + assertThat(config.disk_full_factor(), is(0.001)); + } + + @Test + public void config_override_on_root_is_visible_on_content_cluster() throws Exception { + ContentCluster cluster = model.getContentClusters().get("storage"); + + StorMemfilepersistenceConfig config = model.getConfig(StorMemfilepersistenceConfig.class, cluster.getConfigId()); + assertThat(config.disk_full_factor(), is(0.001)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java new file mode 100644 index 00000000000..e9da8dd2376 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -0,0 +1,298 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.DispatchGroup; +import com.yahoo.vespa.model.search.SearchInterface; +import com.yahoo.vespa.model.search.SearchNode; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; +import static com.yahoo.vespa.model.search.utils.DispatchUtils.assertEngine; +import static com.yahoo.vespa.model.search.utils.DispatchUtils.getDataset; + + +/** + * Unit tests for hierarchic distribution in an indexed content cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class IndexedHierarchicDistributionTest { + + private ContentCluster addDispatcher(ContentCluster c) { + c.getSearch().getIndexed().addTld(new SimpleConfigProducer(new MockRoot(""), ""), new HostResource(new Host(new MockRoot(""), "mockhost"))); + return c; + } + + private ContentCluster getOneGroupCluster() throws Exception { + String groupXml = " <group>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " <node distribution-key='1' hostalias='mockhost'/>\n" + + " <node distribution-key='2' hostalias='mockhost'/>\n" + + " </group>\n"; + return addDispatcher(createCluster(createClusterXml(groupXml, 2, 2))); + } + + private String getTwoGroupsXml(String partitions) { + return " <group>\n" + + " <distribution partitions='" + partitions + "'/>\n" + + " <group distribution-key='0' name='group0'>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " <node distribution-key='1' hostalias='mockhost'/>\n" + + " <node distribution-key='2' hostalias='mockhost'/>\n" + + " </group>\n" + + " <group distribution-key='1' name='group1'>\n" + + " <node distribution-key='3' hostalias='mockhost'/>\n" + + " <node distribution-key='4' hostalias='mockhost'/>\n" + + " <node distribution-key='5' hostalias='mockhost'/>\n" + + " </group>\n" + + " </group>\n"; + } + + private ContentCluster getTwoGroupsCluster() throws Exception { + return addDispatcher(createCluster(createClusterXml(getTwoGroupsXml("3|*"), 6, 6))); + } + + private ContentCluster getTwoGroupsCluster(int redundancy, int searchableCopies, String partitions) throws Exception { + return addDispatcher(createCluster(createClusterXml(getTwoGroupsXml(partitions), redundancy, searchableCopies))); + } + + private void assertSearchNode(int expRowId, int expPartitionId, int expDistibutionKey, SearchNode node) { + assertEquals(expRowId, node.getNodeSpec().rowId()); + assertEquals(expPartitionId, node.getNodeSpec().partitionId()); + assertEquals(expDistibutionKey, ((ContentNode)node.getServiceLayerService()).getDistributionKey()); + } + + private StorDistributionConfig getStorDistributionConfig(ContentCluster c) { + StorDistributionConfig.Builder b = new StorDistributionConfig.Builder(); + c.getConfig(b); + return new StorDistributionConfig(b); + } + + @Test + public void requireThatSearchNodesAreCorrectWithOneGroup() throws Exception { + ContentCluster c = getOneGroupCluster(); + List<SearchNode> searchNodes = c.getSearch().getSearchNodes(); + + assertEquals(3, searchNodes.size()); + assertSearchNode(0, 0, 0, searchNodes.get(0)); + assertSearchNode(0, 1, 1, searchNodes.get(1)); + assertSearchNode(0, 2, 2, searchNodes.get(2)); + } + + @Test + public void requireThatDispatcherIsCorrectWithOneGroup() throws Exception { + ContentCluster c = getOneGroupCluster(); + PartitionsConfig.Dataset dataset = getDataset(c.getSearch().getIndexed().getTLDs().get(0)); + + assertEquals(3, dataset.numparts()); + assertEquals(PartitionsConfig.Dataset.Querydistribution.AUTOMATIC, dataset.querydistribution()); + List<PartitionsConfig.Dataset.Engine> engines = dataset.engine(); + assertEquals(3, engines.size()); + assertEngine(0, 0, engines.get(0)); + assertEngine(0, 1, engines.get(1)); + assertEngine(0, 2, engines.get(2)); + } + + @Test + public void requireThatActivePerLeafGroupIsDefaultWithOneGroup() throws Exception { + ContentCluster c = getOneGroupCluster(); + assertFalse(getStorDistributionConfig(c).active_per_leaf_group()); + } + + @Test + public void requireThatSearchNodesAreCorrectWithTwoGroups() throws Exception { + ContentCluster c = getTwoGroupsCluster(); + List<SearchNode> searchNodes = c.getSearch().getSearchNodes(); + + assertEquals(6, searchNodes.size()); + assertSearchNode(0, 0, 0, searchNodes.get(0)); + assertSearchNode(0, 1, 1, searchNodes.get(1)); + assertSearchNode(0, 2, 2, searchNodes.get(2)); + assertSearchNode(1, 0, 3, searchNodes.get(3)); + assertSearchNode(1, 1, 4, searchNodes.get(4)); + assertSearchNode(1, 2, 5, searchNodes.get(5)); + } + + @Test + public void requireThatDispatcherIsCorrectWithTwoGroups() throws Exception { + ContentCluster c = getTwoGroupsCluster(); + PartitionsConfig.Dataset dataset = getDataset(c.getSearch().getIndexed().getTLDs().get(0)); + + assertEquals(3, dataset.numparts()); + assertEquals(2, dataset.maxnodesdownperfixedrow()); + assertEquals(PartitionsConfig.Dataset.Querydistribution.FIXEDROW, dataset.querydistribution()); + List<PartitionsConfig.Dataset.Engine> engines = dataset.engine(); + assertEquals(6, engines.size()); + assertEngine(0, 0, engines.get(0)); + assertEngine(1, 0, engines.get(1)); + assertEngine(0, 1, engines.get(2)); + assertEngine(1, 1, engines.get(3)); + assertEngine(0, 2, engines.get(4)); + assertEngine(1, 2, engines.get(5)); + } + + @Test + public void requireThatActivePerLeafGroupIsSetWithTwoGroups() throws Exception { + ContentCluster c = getTwoGroupsCluster(); + assertTrue(getStorDistributionConfig(c).active_per_leaf_group()); + } + + private ContentCluster getIllegalMultipleGroupsLevelCluster() throws Exception { + String groupXml = " <group>\n" + + " <distribution partitions='2|*'/>\n" + + " <group distribution-key='0' name='group0'>\n" + + " <distribution partitions='1|*'/>\n" + + " <group distribution-key='0' name='group00'>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " </group>\n" + + " <group distribution-key='1' name='group01'>\n" + + " <node distribution-key='1' hostalias='mockhost'/>\n" + + " </group>\n" + + " </group>\n" + + " </group>\n"; + return createCluster(createClusterXml(groupXml, 2, 2)); + } + + private String getOddGroupsClusterXml() throws Exception { + return " <group>\n" + + " <distribution partitions='2|*'/>\n" + + " <group distribution-key='0' name='group0'>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " </group>\n" + + " <group distribution-key='1' name='group1'>\n" + + " <node distribution-key='1' hostalias='mockhost'/>\n" + + " <node distribution-key='2' hostalias='mockhost'/>\n" + + " </group>\n" + + " </group>\n"; + } + private ContentCluster getIllegalGroupsCluster() throws Exception { + return createCluster(createClusterXml(getOddGroupsClusterXml(), 4, 4)); + } + + private String getRandomDispatchXml() { + return "<tuning>" + + " <dispatch>" + + " <dispatch-policy>random</dispatch-policy>" + + " </dispatch>" + + "</tuning>"; + } + + private ContentCluster getOddGroupsCluster() throws Exception { + String groupXml = " <group>\n" + + " <distribution partitions='2|*'/>\n" + + " <group distribution-key='0' name='group0'>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " <node distribution-key='1' hostalias='mockhost'/>\n" + + " </group>\n" + + " <group distribution-key='1' name='group1'>\n" + + " <node distribution-key='3' hostalias='mockhost'/>\n" + + " <node distribution-key='4' hostalias='mockhost'/>\n" + + " <node distribution-key='5' hostalias='mockhost'/>\n" + + " </group>\n" + + " </group>\n"; + return createCluster(createClusterXml(groupXml, Optional.of(getRandomDispatchXml()), 4, 4)); + } + + @Test + public void requireThatWeMustHaveOnlyOneGroupLevel() { + try { + getIllegalMultipleGroupsLevelCluster(); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("sub group 'group0' contains 2 sub groups.")); + } + } + + @Test + public void requireThatLeafGroupsMustHaveEqualNumberOfNodes() { + try { + getIllegalGroupsCluster(); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("leaf group 'group0' contains 1 node(s) while leaf group 'group1' contains 2 node(s)")); + } + } + + @Test + public void requireThatLeafGroupsCanHaveUnequalNumberOfNodesIfRandomPolicy() throws Exception { + ContentCluster c = getOddGroupsCluster(); + DispatchGroup dg = c.getSearch().getIndexed().getRootDispatch(); + assertEquals(8, dg.getRowBits()); + assertEquals(3, dg.getNumPartitions()); + assertEquals(true, dg.useFixedRowInDispatch()); + assertEquals(1, dg.getMaxNodesDownPerFixedRow()); + ArrayList<SearchInterface> list = new ArrayList<>(); + for(SearchInterface si : dg.getSearchersIterable()) { + list.add(si); + } + assertEquals(5, list.size()); + assertEquals(0, list.get(0).getNodeSpec().partitionId()); + assertEquals(0, list.get(0).getNodeSpec().rowId()); + assertEquals(0, list.get(1).getNodeSpec().partitionId()); + assertEquals(1, list.get(1).getNodeSpec().rowId()); + assertEquals(1, list.get(2).getNodeSpec().partitionId()); + assertEquals(0, list.get(2).getNodeSpec().rowId()); + assertEquals(1, list.get(3).getNodeSpec().partitionId()); + assertEquals(1, list.get(3).getNodeSpec().rowId()); + assertEquals(2, list.get(4).getNodeSpec().partitionId()); + assertEquals(1, list.get(4).getNodeSpec().rowId()); + } + + @Test + public void requireThatLeafGroupsCountMustBeAFactorOfRedundancy() { + try { + getTwoGroupsCluster(3, 3, "2|*"); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Expected number of leaf groups (2) to be a factor of redundancy (3)")); + } + } + + @Test + public void requireThatRedundancyPerGroupMustBeIsEqual() { + try { + getTwoGroupsCluster(4, 4, "1|*"); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Expected distribution partitions should be '2|*'")); + } + } + + @Test + public void requireThatReadyCopiesMustBeEqualToRedundancy() { + try { + getTwoGroupsCluster(4, 3, "2|*"); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("Expected equal amount of ready copies per group")); + } + } + + @Test + public void allowLessReadyCopiesThanRedundancy() throws Exception { + getTwoGroupsCluster(4, 2, "2|*"); + } + + @Test + public void allowNoReadyCopies() throws Exception { + // The active one should be indexed anyhow. Setting up no ready copies + getTwoGroupsCluster(4, 0, "2|*"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedSearchNodeNamingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedSearchNodeNamingTest.java new file mode 100644 index 00000000000..8593d4f01b5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedSearchNodeNamingTest.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.SearchNode; +import org.junit.Test; + +import java.util.List; + +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; +import static junit.framework.TestCase.assertEquals; + +/** + * Unit tests for the naming of search nodes base dir and config ids in an indexed content cluster. + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class IndexedSearchNodeNamingTest { + + private ContentCluster getSingleNodeCluster() throws Exception { + String groupXml = " <group>\n" + + " <node distribution-key='3' hostalias='mockhost'/>\n" + + " </group>\n"; + return createCluster(createClusterXml(groupXml, 1, 1)); + } + + private ContentCluster getMultiNodeCluster() throws Exception { + String groupXml = " <group>\n" + + " <node distribution-key='5' hostalias='mockhost'/>\n" + + " <node distribution-key='3' hostalias='mockhost'/>\n" + + " <node distribution-key='7' hostalias='mockhost'/>\n" + + " </group>\n"; + return createCluster(createClusterXml(groupXml, 1, 1)); + } + + private ContentCluster getMultiGroupCluster() throws Exception { + String groupXml = " <group>\n" + + " <distribution partitions='1|*'/>\n" + + " <group distribution-key='3' name='group0'>\n" + + " <node distribution-key='7' hostalias='mockhost'/>\n" + + " <node distribution-key='11' hostalias='mockhost'/>\n" + + " </group>\n" + + " <group distribution-key='5' name='group1'>\n" + + " <node distribution-key='17' hostalias='mockhost'/>\n" + + " <node distribution-key='13' hostalias='mockhost'/>\n" + + " </group>\n" + + " </group>\n"; + return createCluster(createClusterXml(groupXml, 2, 2)); + } + + private void assertBaseDir(String expected, SearchNode node) { + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + node.getConfig(builder); + ProtonConfig cfg = new ProtonConfig(builder); + assertEquals(expected, cfg.basedir()); + } + + private void assertConfigId(String expected, SearchNode node) { + assertEquals(expected, node.getConfigId()); + } + + private void assertSearchNode(String expName, String expId, SearchNode node) { + assertBaseDir(Defaults.getDefaults().vespaHome() + "var/db/vespa/search/cluster.mycluster/" + expName, node); + assertConfigId("mycluster/search/cluster.mycluster/" + expId, node); + } + + @Test + public void requireThatSingleNodeIsNamedAfterDistributionKey() throws Exception { + ContentCluster cluster = getSingleNodeCluster(); + List<SearchNode> nodes = cluster.getSearch().getSearchNodes(); + assertSearchNode("n3", "3", nodes.get(0)); + } + + @Test + public void requireThatMultipleNodesAreNamedAfterDistributionKey() throws Exception { + ContentCluster cluster = getMultiNodeCluster(); + List<SearchNode> nodes = cluster.getSearch().getSearchNodes(); + assertEquals(3, nodes.size()); + assertSearchNode("n5", "5", nodes.get(0)); + assertSearchNode("n3", "3", nodes.get(1)); + assertSearchNode("n7", "7", nodes.get(2)); + } + + @Test + public void requireThatNodesInHierarchicGroupsAreNamedAfterDistributionKey() throws Exception { + ContentCluster cluster = getMultiGroupCluster(); + List<SearchNode> nodes = cluster.getSearch().getSearchNodes(); + assertEquals(4, nodes.size()); + assertSearchNode("n7", "7", nodes.get(0)); + assertSearchNode("n11", "11", nodes.get(1)); + assertSearchNode("n17", "17", nodes.get(2)); + assertSearchNode("n13", "13", nodes.get(3)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java new file mode 100644 index 00000000000..0f62bc4760d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -0,0 +1,292 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.routing.DocumentProtocol; +import com.yahoo.vespa.model.routing.Routing; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * Test for using the content model to create indexed search clusters. + */ +public class IndexedTest extends ContentBaseTest { + private String createVespaServices(String pre, List<String> sdNames, String post, String mode) { + StringBuilder retval = new StringBuilder(); + retval.append(pre); + + + for (String sdName : sdNames) { + retval.append("<document type='" + sdName + "' " + "mode='" + mode + "'/>"); + } + + retval.append(post); + return retval.toString(); + } + private String createProtonIndexedVespaServices(List<String> sdNames) { + String pre = "<?xml version='1.0' encoding='utf-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0'/>" + + " </admin>" + + " <config name='vespa.configdefinition.specialtokens'>" + + " <tokenlist operation='append'>" + + " <name>default</name>" + + " <tokens operation='append'>" + + " <token>dvd+-r</token>" + + " </tokens>" + + " </tokenlist>" + + " </config>" + + " <jdisc version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node0'/>" + + " </nodes>" + + " </jdisc>" + + " <content version='1.0' id='test'>" + + " <redundancy>1</redundancy>" + + " <engine>" + + " <proton>" + + " <visibility-delay>34</visibility-delay>" + + " </proton>" + + " </engine>" + + " <documents>"; + + String post = " </documents>" + + " <group>" + + " <node hostalias='node0' distribution-key='3' />" + + " </group>" + + "</content>" + + "</services>"; + return createVespaServices(pre, sdNames, post, "index"); + } + private String createProtonStreamingVespaServices(List<String> sdNames) { + String pre = "<?xml version='1.0' encoding='utf-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0'/>" + + " </admin>" + + " <jdisc version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node0'/>" + + " </nodes>" + + " </jdisc>" + + " <content version='1.0' id='test'>" + + " <redundancy>1</redundancy>\n" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <documents>"; + String post = + " </documents>" + + " <group>" + + " <node hostalias='node0' distribution-key='3' />" + + " </group>" + + "</content>" + + "</services>"; + return createVespaServices(pre, sdNames, post, "streaming"); + } + + private VespaModel getIndexedVespaModel() throws ParseException, IOException, SAXException { + return getIndexedVespaModelCreator().create(); + } + + private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() throws ParseException, IOException, SAXException { + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds); + } + + private VespaModel getStreamingVespaModel() throws ParseException, IOException, SAXException { + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create(); + } + + @Test + public void requireMultipleDocumentTypes() throws ParseException, IOException, SAXException { + VespaModelCreatorWithMockPkg creator = getIndexedVespaModelCreator(); + VespaModel model = creator.create(); + DeployState deployState = creator.deployState; + IndexedSearchCluster cluster = model.getContentClusters().get("test").getSearch().getIndexed(); + assertEquals(3, cluster.getDocumentDbs().size()); + NewDocumentType type1 = deployState.getDocumentModel().getDocumentManager().getDocumentType("type1"); + NewDocumentType type2 = deployState.getDocumentModel().getDocumentManager().getDocumentType("type2"); + NewDocumentType type3 = deployState.getDocumentModel().getDocumentManager().getDocumentType("type3"); + assertNotNull(type1); + assertNotNull(type2); + assertNotNull(type3); + } + + @Test + public void requireIndexedOnlyServices() throws ParseException, IOException, SAXException { + VespaModel model = getIndexedVespaModel(); + HostResource h = model.getHostSystem().getHosts().get(0); + String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", + "logd", "configproxy","config-sentinel", "filedistributorservice", + "qrserver", "fleetcontroller", "topleveldispatch", "docprocservice", + "storagenode", "searchnode", "distributor", "transactionlogserver"}; + // TODO DomContentBuilderTest.assertServices(h, expectedServices); + Routing routing = model.getRouting(); + assertNotNull(routing); + assertEquals("[]", routing.getErrors().toString()); + assertEquals(1, routing.getProtocols().size()); + DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); + RoutingTableSpec spec = protocol.getRoutingTableSpec(); + assertEquals(2, spec.getNumHops()); + assertEquals("docproc/cluster.test.indexing/chain.indexing", spec.getHop(0).getName()); + assertEquals("indexing", spec.getHop(1).getName()); + + RouteSpec r; + r = spec.getRoute(0); + assertEquals("default", r.getName()); + assertEquals(1, r.getNumHops()); + assertEquals("indexing", r.getHop(0)); + r = spec.getRoute(1); + assertEquals("storage/cluster.test", r.getName()); + assertEquals(1, r.getNumHops()); + assertEquals("route:test", r.getHop(0)); + r = spec.getRoute(2); + assertEquals("test", r.getName()); + assertEquals(1, r.getNumHops()); + assertEquals("[MessageType:test]", r.getHop(0)); + r = spec.getRoute(3); + assertEquals("test-direct", r.getName()); + assertEquals(1, r.getNumHops()); + assertEquals("[Content:cluster=test]", r.getHop(0)); + r = spec.getRoute(4); + assertEquals("test-index", r.getName()); + assertEquals(2, r.getNumHops()); + assertEquals("docproc/cluster.test.indexing/chain.indexing", r.getHop(0)); + assertEquals("[Content:cluster=test]", r.getHop(1)); + } + @Test + public void requireProtonStreamingOnly() throws ParseException, IOException, SAXException + { + VespaModel model = getStreamingVespaModel(); + HostResource h = model.getHostSystem().getHosts().get(0); + String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", + "logd", "configproxy","config-sentinel", "filedistributorservice", + "qrserver", "storagenode", "searchnode", "distributor", + "transactionlogserver"}; +// TODO DomContentBuilderTest.assertServices(h, expectedServices); + ContentCluster s = model.getContentClusters().get("test"); + assertFalse(s.getSearch().hasIndexedCluster()); + + + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + s.getStorageNodes().getConfig(builder); + s.getStorageNodes().getChildren().get("3").getConfig(builder); + assertTrue(new StorServerConfig(builder).persistence_provider().rpc().connectspec().startsWith("tcp/localhost:191")); + } + + @Test + public void requireCorrectClusterList() throws ParseException, IOException, SAXException + { + VespaModel model = getStreamingVespaModel(); + ContentCluster s = model.getContentClusters().get("test"); + assertNotNull(s); + assertFalse(s.getSearch().hasIndexedCluster()); + ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID); + assertThat(config.storage().size(), is(1)); + assertThat(config.storage(0).name(), is("test")); + assertThat(config.storage(0).configid(), is("test")); + } + + @Test + public void testContentSummaryStore() throws ParseException, IOException, SAXException { + String services= + "<services version='1.0'>" + + "<admin version='2.0'><adminserver hostalias='node0' /></admin>" + + "<content id='docstore' version='1.0'>\n" + + " <redundancy>1</redundancy>\n" + + " <documents>\n" + + " <document mode='index' type='docstorebench'/>\n" + + " </documents>\n" + + " <group>\n" + + " <node distribution-key='0' hostalias='node0'/>\n" + + " </group>\n" + + " <engine>\n" + + " <proton>\n" + + " <searchable-copies>1</searchable-copies>\n" + + " <tuning>\n" + + " <searchnode>\n" + + " <summary>\n" + + " <store>\n" + + " <logstore>\n" + + " <chunk>\n" + + " <maxsize>2048</maxsize>\n" + + " </chunk>\n" + + " </logstore>\n" + + " </store>\n" + + " </summary>\n" + + " </searchnode>\n" + + " </tuning>\n" + + " </proton>\n" + + " </engine>\n" + + " </content>\n" + + " </services>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("docstorebench"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + model.getConfig(pb, "docstore/search/cluster.docstore/0"); + } + + @Test + public void testMixedIndexAndStoreOnly() throws ParseException, IOException, SAXException { + String services= + "<services version='1.0'>" + + " <admin version='2.0'><adminserver hostalias='node0' /></admin>" + + " <content id='docstore' version=\"1.0\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"index_me\" mode=\"index\"/>" + + " <document type=\"store_me\" mode=\"store-only\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + " </content>" + + "</services>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("index_me", "store_me"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + model.getConfig(pb, "docstore/search/cluster.docstore/0"); + ProtonConfig protonConfig = new ProtonConfig(pb); + assertEquals(2, protonConfig.documentdb().size()); + assertEquals("index_me", protonConfig.documentdb(0).inputdoctypename()); + assertEquals("docstore/search/cluster.docstore/index_me", protonConfig.documentdb(0).configid()); + assertEquals("store_me", protonConfig.documentdb(1).inputdoctypename()); + assertEquals("docstore/search", protonConfig.documentdb(1).configid()); + } + + @Test + public void requireThatIndexingDocprocGetsConfigIdBasedOnDistributionKey() throws ParseException, IOException, SAXException { + VespaModel model = getIndexedVespaModel(); + ContainerCluster cluster = model.getContainerClusters().get("cluster.test.indexing"); + assertEquals("docproc/cluster.test.indexing/3", cluster.getContainers().get(0).getConfigId()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java new file mode 100644 index 00000000000..bfb1eb2180a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -0,0 +1,507 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.messagebus.routing.*; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.routing.DocumentProtocol; +import com.yahoo.vespa.model.routing.Protocol; +import com.yahoo.vespa.model.routing.Routing; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.13 + */ +public class IndexingAndDocprocRoutingTest extends ContentBaseTest { + @Test + public void oneContentOneDoctypeImplicitIndexingClusterImplicitIndexingChain() + throws IOException, SAXException, ParseException { + final String CLUSTERNAME = "musiccluster"; + SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, null, null); + searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(searchCluster)); + assertIndexing(model, new DocprocClusterSpec(CLUSTERNAME + ".indexing", new DocprocChainSpec("docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"))); + assertFeedingRoute(model, CLUSTERNAME, "docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"); + } + + @Test + public void oneContentTwoDoctypesImplicitIndexingClusterImplicitIndexingChain() + throws IOException, SAXException, ParseException { + final String CLUSTERNAME = "musicandbookscluster"; + SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, null, null); + searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + searchCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); + VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(searchCluster)); + assertIndexing(model, new DocprocClusterSpec(CLUSTERNAME + ".indexing", new DocprocChainSpec("docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"))); + assertFeedingRoute(model, CLUSTERNAME, "docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"); + } + + @Test + public void twoContentTwoDoctypesImplicitIndexingClusterImplicitIndexingChain() + throws IOException, SAXException, ParseException { + final String MUSIC = "musiccluster"; + SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, null, null); + musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + + final String BOOKS = "bookscluster"; + SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, null, null); + booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); + + VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(musicCluster, booksCluster)); + + assertIndexing(model, + new DocprocClusterSpec(MUSIC + ".indexing", new DocprocChainSpec("docproc/cluster." + MUSIC + ".indexing/chain.indexing")), + new DocprocClusterSpec(BOOKS + ".indexing", new DocprocChainSpec("docproc/cluster." + BOOKS + ".indexing/chain.indexing"))); + + assertFeedingRoute(model, MUSIC, "docproc/cluster." + MUSIC + ".indexing/chain.indexing"); + assertFeedingRoute(model, BOOKS, "docproc/cluster." + BOOKS + ".indexing/chain.indexing"); + } + + + @Test + public void oneContentOneDoctypeExplicitIndexingClusterImplicitIndexingChain() + throws IOException, SAXException, ParseException { + final String CLUSTERNAME = "musiccluster"; + SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, "dpcluster", null); + searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + VespaModel model = getIndexedContentVespaModel(Arrays.asList(new DocprocClusterSpec("dpcluster")), Arrays.asList(searchCluster)); + assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.indexing"))); + assertFeedingRoute(model, CLUSTERNAME, "dpcluster/chain.indexing"); + } + + @Test + public void oneSearchOneDoctypeExplicitIndexingClusterExplicitIndexingChain() + throws IOException, SAXException, ParseException { + String xml = + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<services version=\"1.0\">\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"node0\"/> \n" + + " </admin>\n" + + "\n" + + " <content id=\"searchcluster\" version=\"1.0\">\n" + + " <redundancy>2</redundancy>\n" + + " <documents>\n" + + " <document-processing cluster='dpcluster' chain='fooindexing'/>\n" + + " <document type=\"music\" mode=\"index\"/>\n" + + " </documents>\n" + + " <nodes>\n" + + " <node hostalias=\"node0\" distribution-key=\"0\"/>\n" + + " </nodes>\n" + + " </content>\n" + + " \n" + + " <jdisc version='1.0' id='dpcluster'>\n" + + " <document-processing>\n" + + " <chain id='fooindexing' inherits='indexing '/>\n" + + " </document-processing>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " <http>\n" + + " <server id='dpcluster' port='8000'/>\n" + + " </http>\n" + + " </jdisc>\n" + + "</services>\n"; + VespaModel model = getIndexedSearchVespaModel(xml); + assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.fooindexing", "indexing"), + new DocprocChainSpec("dpcluster/chain.indexing"))); + assertFeedingRouteIndexed(model, "searchcluster", "dpcluster/chain.fooindexing"); + } + + @Test + public void twoContentTwoDoctypesExplicitIndexingInSameIndexingCluster() + throws IOException, SAXException, ParseException { + final String MUSIC = "musiccluster"; + SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, "dpcluster", null); + musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + + final String BOOKS = "bookscluster"; + SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, "dpcluster", null); + booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); + + VespaModel model = getIndexedContentVespaModel(Arrays.asList(new DocprocClusterSpec("dpcluster")), + Arrays.asList(musicCluster, booksCluster)); + + assertIndexing(model, new DocprocClusterSpec("dpcluster", new DocprocChainSpec("dpcluster/chain.indexing"))); + assertFeedingRoute(model, MUSIC, "dpcluster/chain.indexing"); + assertFeedingRoute(model, BOOKS, "dpcluster/chain.indexing"); + } + + @Test + public void noContentClustersOneDocprocCluster() throws ParseException, IOException, SAXException { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + " <admin version='2.0'>\n" + + " <adminserver hostalias='node0'/>\n" + + " </admin>\n" + + " <jdisc version='1.0' id='dokprok'>\n" + + " <document-processing />\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " </jdisc>\n" + + "</services>\n"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "title", "artist"); + VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), + services, sds).create(); + assertIndexing(model, new DocprocClusterSpec("dokprok")); + } + + @Test + public void twoContentTwoDoctypesExplicitIndexingInDifferentIndexingClustersExplicitChain() + throws IOException, SAXException, ParseException { + final String MUSIC = "musiccluster"; + SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, "dpmusiccluster", "dpmusicchain"); + musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + + final String BOOKS = "bookscluster"; + SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, "dpbookscluster", "dpbookschain"); + booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); + + DocprocClusterSpec dpMusicCluster = new DocprocClusterSpec("dpmusiccluster", new DocprocChainSpec("dpmusicchain", "indexing")); + DocprocClusterSpec dpBooksCluster = new DocprocClusterSpec("dpbookscluster", new DocprocChainSpec("dpbookschain", "indexing")); + VespaModel model = getIndexedContentVespaModel(Arrays.asList( + dpMusicCluster, + dpBooksCluster), + Arrays.asList( + musicCluster, + booksCluster)); + + //after we generated model, add indexing chains for validation: + dpMusicCluster.chains.clear(); + dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.indexing")); + dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.dpmusicchain")); + + dpBooksCluster.chains.clear(); + dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.indexing")); + dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.dpbookschain")); + + assertIndexing(model, dpMusicCluster, dpBooksCluster); + assertFeedingRoute(model, MUSIC, "dpmusiccluster/chain.dpmusicchain"); + assertFeedingRoute(model, BOOKS, "dpbookscluster/chain.dpbookschain"); + } + + @Test(expected = IllegalArgumentException.class) + public void twoContentTwoDoctypesExplicitIndexingInDifferentIndexingClustersExplicitChainIncorrectInheritance() + throws IOException, SAXException, ParseException { + final String MUSIC = "musiccluster"; + SearchClusterSpec musicCluster = new SearchClusterSpec(MUSIC, "dpmusiccluster", "dpmusicchain"); + musicCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); + + final String BOOKS = "bookscluster"; + SearchClusterSpec booksCluster = new SearchClusterSpec(BOOKS, "dpbookscluster", "dpbookschain"); + booksCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); + + DocprocClusterSpec dpMusicCluster = new DocprocClusterSpec("dpmusiccluster", new DocprocChainSpec("dpmusicchain")); + DocprocClusterSpec dpBooksCluster = new DocprocClusterSpec("dpbookscluster", new DocprocChainSpec("dpbookschain")); + VespaModel model = getIndexedContentVespaModel(Arrays.asList( + dpMusicCluster, + dpBooksCluster), + Arrays.asList( + musicCluster, + booksCluster)); + + //after we generated model, add indexing chains for validation: + dpMusicCluster.chains.clear(); + dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.indexing")); + dpMusicCluster.chains.add(new DocprocChainSpec("dpmusiccluster/chain.dpmusicchain")); + + dpBooksCluster.chains.clear(); + dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.indexing")); + dpBooksCluster.chains.add(new DocprocChainSpec("dpbookscluster/chain.dpbookschain")); + + assertIndexing(model, dpMusicCluster, dpBooksCluster); + assertFeedingRoute(model, MUSIC, "dpmusiccluster/chain.dpmusicchain"); + assertFeedingRoute(model, BOOKS, "dpbookscluster/chain.dpbookschain"); + } + + private void assertIndexing(VespaModel model, DocprocClusterSpec... expectedDocprocClusters) { + Map<String, ContainerCluster> docprocClusters = getDocprocClusters(model); + assertThat(docprocClusters.size(), is(expectedDocprocClusters.length)); + + for (DocprocClusterSpec expectedDocprocCluster : expectedDocprocClusters) { + ContainerCluster docprocCluster = docprocClusters.get(expectedDocprocCluster.name); + assertThat(docprocCluster, not(nullValue())); + assertThat(docprocCluster.getName(), is(expectedDocprocCluster.name)); + ContainerDocproc containerDocproc = docprocCluster.getDocproc(); + assertThat(containerDocproc, not(nullValue())); + List<DocprocChain> chains = containerDocproc.getChains().allChains().allComponents(); + assertThat(chains.size(), is(expectedDocprocCluster.chains.size())); + List<String> actualDocprocChains = new ArrayList<>(); + for (DocprocChain chain : chains) { + actualDocprocChains.add(chain.getServiceName()); + } + List<String> expectedDocprocChainStrings = new ArrayList<>(); + for (DocprocChainSpec spec : expectedDocprocCluster.chains) { + expectedDocprocChainStrings.add(spec.name); + } + + assertThat(actualDocprocChains, hasItems(expectedDocprocChainStrings.toArray(new String[0]))); + } + } + + private Map<String, ContainerCluster> getDocprocClusters(VespaModel model) { + Map<String, ContainerCluster> docprocClusters = new HashMap<>(); + for (ContainerCluster containerCluster : model.getContainerClusters().values()) { + if (containerCluster.getDocproc() != null) { + docprocClusters.put(containerCluster.getName(), containerCluster); + } + + } + return docprocClusters; + } + + private void assertFeedingRoute(VespaModel model, String searchClusterName, String indexingHopName) { + Routing routing = model.getRouting(); + List<Protocol> protocols = routing.getProtocols(); + + DocumentProtocol documentProtocol = null; + for (Protocol protocol : protocols) { + if (protocol instanceof DocumentProtocol) { + documentProtocol = (DocumentProtocol) protocol; + } + } + + assertNotNull(documentProtocol); + + RoutingTable table = new RoutingTable(documentProtocol.getRoutingTableSpec()); + + HopBlueprint indexingHop = table.getHop("indexing"); + + assertThat(indexingHop, not(nullValue())); + + assertThat(indexingHop.getNumDirectives(), is(1)); + assertThat(indexingHop.getDirective(0), instanceOf(PolicyDirective.class)); + assertThat(indexingHop.getDirective(0).toString(), is("[DocumentRouteSelector]")); + //assertThat(indexingHop.getNumRecipients(), is(1)); + //assertThat(indexingHop.getRecipient(0).getServiceName(), is(searchClusterName)); + + Route route = table.getRoute(searchClusterName); + assertNotNull(route); + + assertThat(route.getNumHops(), is(1)); + Hop messageTypeHop = route.getHop(0); + assertThat(messageTypeHop.getNumDirectives(), is(1)); + assertThat(messageTypeHop.getDirective(0), instanceOf(PolicyDirective.class)); + assertThat(messageTypeHop.getDirective(0).toString(), is("[MessageType:" + searchClusterName + "]")); + PolicyDirective messageTypeDirective = (PolicyDirective) messageTypeHop.getDirective(0); + assertThat(messageTypeDirective.getName(), is("MessageType")); + assertThat(messageTypeDirective.getParam(), is(searchClusterName)); + + String indexingRouteName = DocumentProtocol.getIndexedRouteName(model.getContentClusters().get(searchClusterName).getConfigId()); + Route indexingRoute = table.getRoute(indexingRouteName); + + assertThat(indexingRoute.getNumHops(), is(2)); + assertThat(indexingRoute.getHop(0).getServiceName(), is(indexingHopName)); + assertThat(indexingRoute.getHop(1), not(nullValue())); + } + + private void assertFeedingRouteIndexed(VespaModel model, String searchClusterName, String indexingHopName) { + Routing routing = model.getRouting(); + List<Protocol> protocols = routing.getProtocols(); + + DocumentProtocol documentProtocol = null; + for (Protocol protocol : protocols) { + if (protocol instanceof DocumentProtocol) { + documentProtocol = (DocumentProtocol) protocol; + } + } + + assertNotNull(documentProtocol); + + RoutingTable table = new RoutingTable(documentProtocol.getRoutingTableSpec()); + + Route indexingRoute = table.getRoute("searchcluster-index"); + assertThat(indexingRoute.getNumHops(), is(2)); + assertThat(indexingRoute.getHop(0).toString(), is(indexingHopName)); + assertThat(indexingRoute.getHop(1).toString(), is("[Content:cluster=" + searchClusterName + "]")); + } + + + private String createVespaServices(String mainPre, String contentClusterPre, String contentClusterPost, + String searchClusterPre, String searchClusterPost, String searchClusterPostPost, + String mainPost, List<SearchClusterSpec> searchClusterSpecs) { + StringBuilder retval = new StringBuilder(); + retval.append(mainPre); + + for (SearchClusterSpec searchClusterSpec : searchClusterSpecs) { + retval.append(contentClusterPre).append(searchClusterSpec.name).append(contentClusterPost); + retval.append(searchClusterPre); + for (SearchDefSpec searchDefSpec : searchClusterSpec.searchDefs) { + retval.append(" <document type='") + .append(searchDefSpec.typeName) + .append("' mode='") + .append("index") + .append("' />\n"); + } + if (searchClusterSpec.indexingClusterName != null) { + retval.append(" <document-processing cluster='").append(searchClusterSpec.indexingClusterName).append("'"); + if (searchClusterSpec.indexingChainName != null) { + retval.append(" chain='").append(searchClusterSpec.indexingChainName).append("'"); + } + retval.append("/>\n"); + } + retval.append(searchClusterPost); + retval.append(searchClusterPostPost); + } + + retval.append(mainPost); + System.err.println(retval); + return retval.toString(); + } + + private String createVespaServicesWithContent(List<DocprocClusterSpec> docprocClusterSpecs, List<SearchClusterSpec> searchClusterSpecs) { + String mainPre = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + + " <admin version='2.0'>\n" + + " <adminserver hostalias='node0'/>\n" + + " </admin>\n" + + + " <jdisc version='1.0'>\n" + + " <search/>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " </jdisc>\n"; + int clusterNo = 0; + for (DocprocClusterSpec docprocClusterSpec : docprocClusterSpecs) { + String docprocCluster = ""; + docprocCluster += " <jdisc version='1.0' id='" + docprocClusterSpec.name + "'>\n"; + + if (docprocClusterSpec.chains != null && docprocClusterSpec.chains.size() > 0) { + docprocCluster += " <document-processing>\n"; + for (DocprocChainSpec chain : docprocClusterSpec.chains) { + if (chain.inherits.isEmpty()) { + docprocCluster += " <chain id='" + chain.name + "'/>\n"; + } else { + docprocCluster += " <chain id='" + chain.name + "'"; + docprocCluster += " inherits='"; + + for (String inherit : chain.inherits) { + docprocCluster += inherit + " "; + } + + docprocCluster += "'/>\n"; + } + } + docprocCluster += " </document-processing>\n"; + } else { + docprocCluster += " <document-processing/>\n"; + } + + docprocCluster += " <http>\n" + + " <server id='" + docprocClusterSpec.name + "' port='" + (8000 + 10 * clusterNo) + "'/>\n" + + " </http>\n"; + + docprocCluster += " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " </jdisc>\n"; + mainPre += docprocCluster; + clusterNo++; + } + + String contentClusterPre = + " <content version='1.0' id='"; + + String contentClusterPost = "'>\n"; + String searchClusterPre = + " <redundancy>1</redundancy>\n" + + " <documents>\n"; + String searchClusterPost = + " </documents>\n" + + " <group>\n" + + " <node hostalias='node0' distribution-key='0' />\n" + + " </group>\n"; + + String searchClusterPostPost = " </content>\n"; + + String mainPost = + "</services>\n"; + return createVespaServices(mainPre, contentClusterPre, contentClusterPost, searchClusterPre, + searchClusterPost, searchClusterPostPost, mainPost, searchClusterSpecs); + } + + private VespaModel getIndexedSearchVespaModel(String xml) + throws ParseException, IOException, SAXException { + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "album", "artist"); + return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + } + + private VespaModel getIndexedContentVespaModel(List<DocprocClusterSpec> docprocClusterSpecs, List<SearchClusterSpec> searchClusterSpecs) + throws ParseException, IOException, SAXException { + List<String> sds = new ArrayList<>(); + + for (SearchClusterSpec cluster : searchClusterSpecs) { + for (SearchDefSpec def : cluster.searchDefs) { + sds.add(ApplicationPackageUtils.generateSearchDefinition(def.typeName, def.field1Name, def.field2Name)); + } + } + + return new VespaModelCreatorWithMockPkg(getHosts(), + createVespaServicesWithContent(docprocClusterSpecs, searchClusterSpecs), sds).create(); + } + + private class SearchClusterSpec { + private final String name; + private List<SearchDefSpec> searchDefs = new ArrayList<>(2); + private String indexingClusterName; + private String indexingChainName; + + private SearchClusterSpec(String name, String indexingClusterName, String indexingChainName) { + this.name = name; + this.indexingClusterName = indexingClusterName; + this.indexingChainName = indexingChainName; + } + } + + private class SearchDefSpec { + private String typeName; + private String field1Name; + private String field2Name; + + private SearchDefSpec(String typeName, String field1Name, String field2Name) { + this.typeName = typeName; + this.field1Name = field1Name; + this.field2Name = field2Name; + } + } + + private class DocprocClusterSpec { + private final String name; + private final List<DocprocChainSpec> chains = new ArrayList<>(); + + private DocprocClusterSpec(String name, DocprocChainSpec ... chains) { + this.name = name; + this.chains.addAll(Arrays.asList(chains)); + } + } + + private class DocprocChainSpec { + private final String name; + private final List<String> inherits = new ArrayList<>(); + + private DocprocChainSpec(String name, String ... inherits) { + this.name = name; + this.inherits.addAll(Arrays.asList(inherits)); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/SearchCoverageTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/SearchCoverageTest.java new file mode 100644 index 00000000000..777a8269470 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/SearchCoverageTest.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SearchCoverageTest { + + @Test + public void requireThatAccessorWork() { + SearchCoverage coverage = new SearchCoverage.Builder() + .setMinimum(0.1) + .setMinWaitAfterCoverageFactor(0.2) + .setMaxWaitAfterCoverageFactor(0.3) + .build(); + assertEquals(0.1, coverage.getMinimum(), 1E-6); + assertEquals(0.2, coverage.getMinWaitAfterCoverageFactor(), 1E-6); + assertEquals(0.3, coverage.getMaxWaitAfterCoverageFactor(), 1E-6); + } + + @Test + public void requireThatDefaultsAreNull() { + SearchCoverage search = new SearchCoverage.Builder().build(); + assertNull(search.getMinimum()); + assertNull(search.getMinWaitAfterCoverageFactor()); + assertNull(search.getMaxWaitAfterCoverageFactor()); + } + + @Test + public void requireThatInvalidMinimumCanNotBeSet() { + SearchCoverage.Builder coverage = new SearchCoverage.Builder(); + coverage.setMinimum(0.5); + assertEquals(0.5, coverage.build().getMinimum(), 1E-6); + try { + coverage.setMinimum(-0.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got -0.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMinimum(), 1E-6); + try { + coverage.setMinimum(1.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got 1.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMinimum(), 1E-6); + } + + @Test + public void requireThatInvalidMinWaitAfterCoverageFactorCanNotBeSet() { + SearchCoverage.Builder coverage = new SearchCoverage.Builder(); + coverage.setMinWaitAfterCoverageFactor(0.5); + assertEquals(0.5, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMinWaitAfterCoverageFactor(-0.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got -0.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMinWaitAfterCoverageFactor(1.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got 1.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + } + + @Test + public void requireThatInvalidMaxWaitAfterCoverageFactorCanNotBeSet() { + SearchCoverage.Builder coverage = new SearchCoverage.Builder(); + coverage.setMaxWaitAfterCoverageFactor(0.5); + assertEquals(0.5, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMaxWaitAfterCoverageFactor(-0.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got -0.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMaxWaitAfterCoverageFactor(1.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected value in range [0, 1], got 1.5.", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + } + + @Test + public void requireThatMinWaitCanNotBeSetLargerThanMaxWait() { + SearchCoverage.Builder coverage = new SearchCoverage.Builder(); + coverage.setMaxWaitAfterCoverageFactor(0.5); + coverage.setMinWaitAfterCoverageFactor(0.4); + assertEquals(0.4, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + coverage.setMinWaitAfterCoverageFactor(0.5); + assertEquals(0.5, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMinWaitAfterCoverageFactor(0.6); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Minimum wait (got 0.6) must be no larger than maximum wait (was 0.5).", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMinWaitAfterCoverageFactor(), 1E-6); + } + + @Test + public void requireThatMaxWaitCanNotBeSetSmallerThanMaxWait() { + SearchCoverage.Builder coverage = new SearchCoverage.Builder(); + coverage.setMinWaitAfterCoverageFactor(0.5); + coverage.setMaxWaitAfterCoverageFactor(0.6); + assertEquals(0.6, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + coverage.setMaxWaitAfterCoverageFactor(0.5); + assertEquals(0.5, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + try { + coverage.setMaxWaitAfterCoverageFactor(0.4); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Maximum wait (got 0.4) must be no smaller than minimum wait (was 0.5).", e.getMessage()); + } + assertEquals(0.5, coverage.build().getMaxWaitAfterCoverageFactor(), 1E-6); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java new file mode 100644 index 00000000000..95761de7331 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -0,0 +1,360 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; +import com.yahoo.vespa.config.content.core.StorVisitorConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.PersistenceConfig; +import com.yahoo.vespa.config.storage.StorDevicesConfig; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.storagecluster.StorageCluster; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import static org.junit.Assert.*; + +public class StorageClusterTest { + + StorageCluster parse(String xml) { + MockRoot root = new MockRoot(); + root.getDeployState().getDocumentModel().getDocumentManager().add( + new NewDocumentType(new NewDocumentType.Name("music")) + ); + root.getDeployState().getDocumentModel().getDocumentManager().add( + new NewDocumentType(new NewDocumentType.Name("movies")) + ); + Document doc = XML.getDocument(xml); + Element clusterElem = doc.getDocumentElement(); + ContentCluster cluster = new ContentCluster.Builder(null, null).build(root, clusterElem); + + root.freezeModelTopology(); + return cluster.getStorageNodes(); + } + + @Test + public void testBasics() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse("<content id=\"foofighters\"><documents/>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>\n"). + getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(false, config.is_distributor()); + assertEquals("foofighters", config.cluster_name()); + } + + @Test + public void testMerges() { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + parse("" + + "<content id=\"foofighters\">\n" + + " <documents/>" + + " <tuning>" + + " <merges max-per-node=\"1K\" max-queue-size=\"10K\"/>\n" + + " </tuning>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>" + ).getConfig(builder); + + StorServerConfig config = new StorServerConfig(builder); + assertEquals(1024, config.max_merges_per_node()); + assertEquals(1024*10, config.max_merge_queue_size()); + } + + @Test + public void testVisitors() { + StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); + parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>\n" + + " <visitors thread-count=\"7\" max-queue-size=\"1000\">\n" + + " <max-concurrent fixed=\"42\" variable=\"100\"/>\n" + + " </visitors>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>" + ).getConfig(builder); + + StorVisitorConfig config = new StorVisitorConfig(builder); + assertEquals(42, config.maxconcurrentvisitors_fixed()); + assertEquals(100, config.maxconcurrentvisitors_variable()); + assertEquals(7, config.visitorthreads()); + assertEquals(1000, config.maxvisitorqueuesize()); + } + + @Test + public void testPersistenceThreads() { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <engine>" + + " <vds/>" + + " </engine>" + + " <tuning>\n" + + " <persistence-threads>\n" + + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + + " <thread count=\"1\"/>\n" + + " </persistence-threads>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>" + ).getConfig(builder); + + StorFilestorConfig config = new StorFilestorConfig(builder); + + assertEquals(4, config.threads().size()); + assertEquals(190, config.threads().get(0).lowestpri()); + assertEquals(190, config.threads().get(1).lowestpri()); + assertEquals(60, config.threads().get(2).lowestpri()); + assertEquals(255, config.threads().get(3).lowestpri()); + + assertEquals(true, config.enable_multibit_split_optimalization()); + } + + @Test + public void testNoPersistenceThreads() { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>" + ).getConfig(builder); + + StorFilestorConfig config = new StorFilestorConfig(builder); + + assertEquals(0, config.threads().size()); + } + + @Test + public void testMaintenance() { + StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); + parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>" + + " <maintenance start=\"01:00\" stop=\"02:00\" high=\"tuesday\"/>\n" + + " </tuning>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>" + ).getConfig(builder); + StorIntegritycheckerConfig config = new StorIntegritycheckerConfig(builder); + + assertEquals(60, config.dailycyclestart()); + assertEquals(120, config.dailycyclestop()); + assertEquals("rrRrrrr", config.weeklycycle()); + } + + @Test + public void testCapacity() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\" capacity=\"1.5\"/>\n" + + " <node distribution-key=\"2\" hostalias=\"mockhost\" capacity=\"2.0\"/>\n" + + " </group>\n" + + "</cluster>" + ); + + ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + + for (int i = 0; i < 3; ++i) { + StorageNode node = cluster.getStorageNodes().getChildren().get("" + i); + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + node.getConfig(builder); + StorServerConfig config = new StorServerConfig(builder); + assertEquals(1.0 + (double)i * 0.5, config.node_capacity(), 0.001); + } + } + + @Test + public void testRootFolder() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</cluster>" + ); + + ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + + StorageNode node = cluster.getStorageNodes().getChildren().get("0"); + + { + StorDevicesConfig.Builder builder = new StorDevicesConfig.Builder(); + node.getConfig(builder); + StorDevicesConfig config = new StorDevicesConfig(builder); + assertEquals(Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/storage/storage/0", config.root_folder()); + } + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + node.getConfig(builder); + StorServerConfig config = new StorServerConfig(builder); + assertEquals(Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/storage/storage/0", config.root_folder()); + } + + { + StorServerConfig.Builder builder = new StorServerConfig.Builder(); + cluster.getDistributorNodes().getConfig(builder); + cluster.getDistributorNodes().getChildren().get("0").getConfig(builder); + StorServerConfig config = new StorServerConfig(builder); + assertEquals(Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/storage/distributor/0", config.root_folder()); + } + } + + @Test + public void testGenericPersistenceTuning() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + "<documents/>" + + "<engine>\n" + + " <fail-partition-on-error>true</fail-partition-on-error>\n" + + " <revert-time>34m</revert-time>\n" + + " <recovery-time>5d</recovery-time>\n" + + "</engine>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + "</cluster>" + ); + + ContentCluster cluster = new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + + PersistenceConfig.Builder builder = new PersistenceConfig.Builder(); + cluster.getStorageNodes().getConfig(builder); + + PersistenceConfig config = new PersistenceConfig(builder); + assertEquals(true, config.fail_partition_on_error()); + assertEquals(34 * 60, config.revert_time_period()); + assertEquals(5 * 24 * 60 * 60, config.keep_remove_time_period()); + } + + @Test + public void requireThatUserDoesntSpecifyBothGroupAndNodes() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + "<engine>\n" + + " <fail-partition-on-error>true</fail-partition-on-error>\n" + + " <revert-time>34m</revert-time>\n" + + " <recovery-time>5d</recovery-time>\n" + + "</engine>" + + " <group>\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " <nodes>\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </nodes>\n" + + "</cluster>" + ); + + try { + new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + assertTrue(false); + } catch (Exception e) { + + } + } + + @Test + public void requireThatGroupNamesMustBeUniqueAmongstSiblings() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + "<documents/>\n" + + " <group>\n" + + " <distribution partitions=\"*\"/>\n" + + " <group distribution-key=\"0\" name=\"bar\">\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " <group distribution-key=\"0\" name=\"bar\">\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " </group>\n" + + "</cluster>" + ); + try { + new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + fail("Did not get exception with duplicate group names"); + } catch (RuntimeException e) { + assertEquals("Cluster 'storage' has multiple groups with name 'bar' in the same subgroup. " + + "Group sibling names must be unique.", e.getMessage()); + } + } + + @Test + public void requireThatGroupNamesCanBeDuplicatedAcrossLevels() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + "<documents/>\n" + + " <group>\n" + + " <distribution partitions=\"*\"/>\n" + + " <group distribution-key=\"0\" name=\"bar\">\n" + + " <group distribution-key=\"0\" name=\"foo\">\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " </group>\n" + + " <group distribution-key=\"0\" name=\"foo\">\n" + + " <group distribution-key=\"0\" name=\"bar\">\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " </group>\n" + + " </group>\n" + + "</cluster>" + ); + // Should not throw. + new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + } + + @Test + public void requireThatNestedGroupsRequireDistribution() { + Document doc = XML.getDocument( + "<cluster id=\"storage\">\n" + + "<documents/>\n" + + " <group>\n" + + " <group distribution-key=\"0\" name=\"bar\">\n" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " <group distribution-key=\"0\" name=\"baz\">\n" + + " <node distribution-key=\"1\" hostalias=\"mockhost\"/>\n" + + " </group>\n" + + " </group>\n" + + "</cluster>" + ); + try { + new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + fail("Did not get exception with missing distribution element"); + } catch (RuntimeException e) { + assertEquals("'distribution' attribute is required with multiple subgroups", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java new file mode 100644 index 00000000000..95b5d273291 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java @@ -0,0 +1,177 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.routing.DocumentProtocol; +import com.yahoo.vespa.model.routing.Routing; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +public class StorageContentTest extends ContentBaseTest { + // TODO: Test with document-definitions + + private String createStorageVespaServices(String cluster1docs, String cluster2docs) { + return "<?xml version='1.0' encoding='utf-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0'/>" + + " </admin>" + + " <content version='1.0' id='bar'>" + + " <redundancy>1</redundancy>\n" + + cluster1docs + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + " </content>" + + " <content version='1.0' id='zoo'>" + + " <redundancy>1</redundancy>\n" + + cluster2docs + + " <group>" + + " <node hostalias='node0' distribution-key='0' />" + + " </group>" + + "</content>" + + "</services>"; + } + + private VespaModel getStorageVespaModel(String cluster1docs, String cluster2docs) throws ParseException, IOException, SAXException { + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create(); + } + + public void doTestRouting(String cluster1docs, String cluster2docs, String expectedRoutes) throws Exception { + VespaModel model = getStorageVespaModel(cluster1docs, cluster2docs); + + if (expectedRoutes == null) { + return; + } + + Routing routing = model.getRouting(); + assertNotNull(routing); + + assertEquals(0, routing.getErrors().size()); + assertEquals(1, routing.getProtocols().size()); + DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); + + RoutingTableSpec spec = protocol.getRoutingTableSpec(); + assertEquals(1, spec.getNumHops()); + assertEquals("indexing", spec.getHop(0).getName()); + assertEquals("[DocumentRouteSelector]", spec.getHop(0).getSelector()); + + Map<String, RouteSpec> routes = new TreeMap<>(); + + for (int i = 0; i < spec.getNumRoutes(); ++i) { + RouteSpec r = spec.getRoute(i); + + routes.put(r.getName(), r); + } + + { + RouteSpec r = routes.get("default"); + assertEquals(1, r.getNumHops()); + assertEquals("indexing", r.getHop(0)); + } + + Set<String> configuredRoutes = new TreeSet<>(); + + DocumentrouteselectorpolicyConfig.Builder builder = new DocumentrouteselectorpolicyConfig.Builder(); + protocol.getConfig(builder); + DocumentrouteselectorpolicyConfig config = new DocumentrouteselectorpolicyConfig(builder); + + for (DocumentrouteselectorpolicyConfig.Route r : config.route()) { + configuredRoutes.add(r.name() + " : " + r.selector()); + } + + StringBuilder routeStr = new StringBuilder(); + for (String r : configuredRoutes) { + routeStr.append(r).append('\n'); + } + + assertEquals(expectedRoutes, routeStr.toString()); + } + + @Test + public void testDocumentTypesRouting() throws Exception { + String cluster1docs = "<documents>\n" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " <document type=\"type2\" mode=\"store-only\"/>\n" + + "</documents>\n"; + String cluster2docs = "<documents>\n" + + " <document type=\"type3\" mode=\"store-only\"/>\n" + + "</documents>\n"; + String expectedRoutes = "bar : (type1) OR (type2)\n" + + "zoo : (type3)\n"; + + doTestRouting(cluster1docs, cluster2docs, expectedRoutes); + } + + @Test + public void testDocumentTypesAndLocalSelectionRouting() throws Exception { + String cluster1docs = "<documents>\n" + + " <document type=\"type1\" mode=\"store-only\" selection=\"1 != 2\"/>\n" + + " <document type=\"type2\" mode=\"store-only\" selection=\"now() > 1000\"/>\n" + + "</documents>\n"; + String cluster2docs = "<documents>\n" + + " <document type=\"type3\" mode=\"store-only\" selection=\"true\"/>\n" + + "</documents>\n"; + String expectedRoutes = "bar : (type1 AND (1 != 2)) OR (type2 AND (now() > 1000))\n" + + "zoo : (type3 AND (true))\n"; + + doTestRouting(cluster1docs, cluster2docs, expectedRoutes); + } + + @Test + public void testDocumentTypesAndGlobalSelection() throws Exception { + String cluster1docs = "<documents selection=\"5 != 6\">\n" + + " <document type=\"type1\" mode=\"store-only\" selection=\"type1.f1 == 'baz'\"/>\n" + // Can refer to own type + " <document type=\"type2\" mode=\"store-only\"/>\n" + + "</documents>\n"; + String cluster2docs = "<documents selection=\"true\">\n" + + " <document type=\"type3\" mode=\"store-only\"/>\n" + + "</documents>\n"; + String expectedRoutes = "bar : (5 != 6) AND ((type1 AND (type1.f1 == 'baz')) OR (type2))\n" + + "zoo : (true) AND ((type3))\n"; + + doTestRouting(cluster1docs, cluster2docs, expectedRoutes); + } + + @Test + public void testIllegalDocumentTypesInSelection() throws Exception { + String localDefs = "<documents>\n" + + " <document type=\"type1\" mode=\"store-only\"/>\n" + + " <document type=\"type2\" mode=\"store-only\" selection=\"type1.bar == 'baz'\"/>\n" + // Not own type + "</documents>\n"; + String globalDefs = "<documents selection=\"type3.foo\">\n" + // No doctypes allowed + " <document type=\"type3\" mode=\"store-only\"/>\n" + + "</documents>\n"; + String expectedRoutes = null; + + try { + // Local + doTestRouting(localDefs, localDefs, expectedRoutes); + fail("no exception thrown for doc type in local selection"); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("Selection for document type 'type2" + + "' can not contain references to other " + + "document types (found reference to type 'type1')")); + } + + try { + // Global + doTestRouting(globalDefs, globalDefs, expectedRoutes); + fail("no exception thrown for doc type in global selection"); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("Document type references are not allowed")); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java new file mode 100644 index 00000000000..da6636255ba --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import org.junit.Test; +import org.w3c.dom.Document; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test for storage groups. + */ +public class StorageGroupTest { + ContentCluster parse(String xml) { + Document doc = XML.getDocument(xml); + return new ContentCluster.Builder(null, null).build(new MockRoot(), doc.getDocumentElement()); + } + + @Test + public void testSingleGroup() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + ContentCluster cluster = parse( + "<content id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <node jvmargs=\"foo\" hostalias=\"mockhost\" distribution-key=\"0\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"1\"/>\n" + + " </group>\n" + + "</content>" + ); + + cluster.getConfig(builder); + + assertEquals("content", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustertype")); + assertEquals("storage", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustername")); + assertEquals("0", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("index")); + + assertEquals("content", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("clustertype")); + assertEquals("storage", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("clustername")); + assertEquals("0", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("index")); + + StorDistributionConfig config = new StorDistributionConfig(builder); + + assertEquals(1, config.group().size()); + assertEquals("invalid", config.group(0).index()); + assertEquals("invalid", config.group(0).name()); + assertEquals(2, config.group(0).nodes().size()); + assertEquals(0, config.group(0).nodes(0).index()); + assertEquals(1, config.group(0).nodes(1).index()); + //assertNotNull(cluster.getRootGroup().getNodes().get(0).getHost()); + } + + @Test + public void testNestedGroupsNoDistribution() { + try { + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <group distribution-key=\"0\" name=\"base\">\n" + + " <group distribution-key=\"0\" name=\"sub1\">\n" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"1\"/>\n" + + " </group>\n" + + " <group distribution-key=\"1\" name=\"sub2\">\n" + + " <node hostalias=\"mockhost\" distribution-key=\"2\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"3\"/>\n" + + " </group>\n" + + " </group>\n" + + "</cluster>" + ); + assertTrue(false); + } catch (Exception e) { + } + } + + @Test + public void testNestedGroups() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <distribution partitions=\"1|*\"/>\n" + + " <group distribution-key=\"0\" name=\"sub1\">\n" + + " <node hostalias=\"mockhost\" distribution-key=\"0\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"1\"/>\n" + + " </group>\n" + + " <group distribution-key=\"1\" name=\"sub2\">\n" + + " <distribution partitions=\"1|*\"/>\n" + + " <group distribution-key=\"0\" name=\"sub3\">\n" + + " <node hostalias=\"mockhost\" distribution-key=\"2\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"3\"/>\n" + + " </group>\n" + + " <group distribution-key=\"1\" name=\"sub4\">\n" + + " <node hostalias=\"mockhost\" distribution-key=\"4\"/>\n" + + " <node hostalias=\"mockhost\" distribution-key=\"5\"/>\n" + + " </group>\n" + + " </group>\n" + + " </group>\n" + + "</content>" + ).getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + + assertEquals(5, config.group().size()); + assertEquals("invalid", config.group(0).index()); + assertEquals("0", config.group(1).index()); + assertEquals("1", config.group(2).index()); + assertEquals("1.0", config.group(3).index()); + assertEquals("1.1", config.group(4).index()); + assertEquals("invalid", config.group(0).name()); + assertEquals("sub1", config.group(1).name()); + assertEquals("sub2", config.group(2).name()); + assertEquals("sub3", config.group(3).name()); + assertEquals("sub4", config.group(4).name()); + assertEquals(2, config.group(1).nodes().size()); + assertEquals(0, config.group(1).nodes(0).index()); + assertEquals(1, config.group(1).nodes(1).index()); + assertEquals(0, config.group(2).nodes().size()); + assertEquals(2, config.group(3).nodes().size()); + assertEquals(2, config.group(3).nodes(0).index()); + assertEquals(3, config.group(3).nodes(1).index()); + assertEquals(2, config.group(4).nodes().size()); + assertEquals(4, config.group(4).nodes(0).index()); + assertEquals(5, config.group(4).nodes(1).index()); + + assertEquals("1|*", config.group(0).partitions()); + } + + @Test + public void testGroupCapacity() { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + parse( + "<content version=\"1.0\" id=\"storage\">\n" + + " <documents/>" + + " <group>\n" + + " <distribution partitions=\"1|*\"/>\n" + + " <group distribution-key=\"0\" name=\"sub1\">\n" + + " <node hostalias=\"mockhost\" capacity=\"0.5\" distribution-key=\"0\"/>\n" + + " <node hostalias=\"mockhost\" capacity=\"1.5\" distribution-key=\"1\"/>\n" + + " </group>\n" + + " <group distribution-key=\"1\" name=\"sub2\">\n" + + " <node hostalias=\"mockhost\" capacity=\"2.0\" distribution-key=\"2\"/>\n" + + " <node hostalias=\"mockhost\" capacity=\"1.5\" distribution-key=\"3\"/>\n" + + " </group>\n" + + " </group>\n" + + "</content>" + ).getConfig(builder); + + StorDistributionConfig config = new StorDistributionConfig(builder); + + assertEquals(3, config.group().size()); + assertEquals(5.5, config.group(0).capacity(), 0.001); + assertEquals(2, config.group(1).capacity(), 0.001); + assertEquals(3.5, config.group(2).capacity(), 0.001); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java new file mode 100644 index 00000000000..b0f2214d058 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageNodeTest.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.config.storage.StorDevicesConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author hakon + */ +public class StorageNodeTest { + private StorDevicesConfig getConfig(boolean useVdsEngine) { + String vdsConfig = useVdsEngine ? " <engine>" + + " <vds/>" + + " </engine>" : ""; + + String servicesXml = "<?xml version='1.0' encoding='utf-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0'/>" + + " </admin>" + + " <content version='1.0' id='zoo'>" + + " <redundancy>1</redundancy>" + + " <nodes count='1' />" + + " <documents>" + + " <document type='type1' mode='streaming' />" + + " </documents>" + + vdsConfig + + " </content>" + + "</services>"; + List<String> searchDefinitions = ApplicationPackageUtils.generateSearchDefinition("type1"); + VespaModelCreatorWithMockPkg modelCreator = + new VespaModelCreatorWithMockPkg(null, servicesXml, searchDefinitions); + ApplicationPackage appPkg = modelCreator.appPkg; + boolean failOnOutOfCapacity = true; + InMemoryProvisioner provisioner = + new InMemoryProvisioner(failOnOutOfCapacity, "host1.yahoo.com", "host2.yahoo.com"); + DeployProperties.Builder builder = new DeployProperties.Builder(); + DeployProperties properties = builder.hostedVespa(true).build(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(appPkg) + .modelHostProvisioner(provisioner) + .properties(properties) + .build(); + VespaModel model = modelCreator.create(true, deployState); + return model.getConfig(StorDevicesConfig.class, "zoo/storage/0"); + } + + @Test + public void verifyDiskPathConfigIsSetForVds() throws Exception { + StorDevicesConfig config = getConfig(true); + assertEquals(1, config.disk_path().size()); + assertEquals(Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/zoo/storage/0/disks/d0", config.disk_path(0)); + } + + @Test + public void verifyDiskPathConfigIsNotSetForNonHosted() throws Exception { + StorDevicesConfig config = getConfig(false); + assertEquals(0, config.disk_path().size()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java new file mode 100644 index 00000000000..2f692f40b0e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class TuningDispatchTest { + + @Test + public void requireThatAccessorWork() { + TuningDispatch dispatch = new TuningDispatch.Builder() + .setMaxHitsPerPartition(69) + .setDispatchPolicy("round-robin") + .setMinGroupCoverage(7.5) + .setMinActiveDocsCoverage(12.5) + .build(); + assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); + assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); + assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); + assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + } + @Test + public void requireThatRandomDispatchWork() { + TuningDispatch dispatch = new TuningDispatch.Builder() + .setDispatchPolicy("random") + .build(); + assertTrue(TuningDispatch.DispatchPolicy.RANDOM == dispatch.getDispatchPolicy()); + assertNull(dispatch.getMinGroupCoverage()); + assertNull(dispatch.getMinActiveDocsCoverage()); + } + + @Test + public void requireThatDefaultsAreNull() { + TuningDispatch dispatch = new TuningDispatch.Builder().build(); + assertNull(dispatch.getMaxHitsPerPartition()); + assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/VDSProviderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/VDSProviderTest.java new file mode 100644 index 00000000000..bbbb50ce2c3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/VDSProviderTest.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.engines.VDSEngine; +import org.junit.Ignore; +import org.junit.Test; +import org.w3c.dom.Document; + +import static org.junit.Assert.assertEquals; + +public class VDSProviderTest { + VDSEngine parse(String xml) { + Document doc = XML.getDocument(xml); + return new VDSEngine(null, new ModelElement(doc.getDocumentElement())); + } + + @Test + public void testTuning() { + StorMemfilepersistenceConfig.Builder builder = new StorMemfilepersistenceConfig.Builder(); + + parse( + " <vds>\n" + + " <tuning>\n" + + " <disk-full-ratio>0.93</disk-full-ratio>\n" + + " <cache-size>1G</cache-size>\n" + + " </tuning>" + + "</vds>" + ).getConfig(builder); + + StorMemfilepersistenceConfig config = new StorMemfilepersistenceConfig(builder); + + assertEquals(0.93, config.disk_full_factor(), 0.01); + assertEquals(1024 * 1024 * 1024, config.cache_size()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/YamasConfigSnoopTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/YamasConfigSnoopTest.java new file mode 100644 index 00000000000..103c6339fe9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/YamasConfigSnoopTest.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.metrics.MetricsmanagerConfig; +import org.junit.Test; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + + +/** + * @author havardpe + **/ +public class YamasConfigSnoopTest { + + private TestRoot root; + + public void initRoot(int interval) throws Exception { + TestDriver tester = new TestDriver(); + root = tester.buildModel(getAdminXml(interval) + getContent()); + } + + private String getAdminXml(int interval) { + return "" + + "<admin version='2.0'>" + + " <adminserver hostalias='mockhost' />" + + " <yamas interval='" + interval + "' systemname='test' />" + + "</admin>"; + } + + private String getContent() { + return ( + "<content version='1.0' id='search'>"+ + " <documents/>"+ + " <nodes>"+ + " <node hostalias='mockhost' distribution-key='0' />"+ + " </nodes>"+ + "</content>"); + } + + private MetricsmanagerConfig getConfig() { + return root.getConfig(MetricsmanagerConfig.class, "search/storage/0"); + } + + @Test + public void correct_config_is_snooped() throws Exception { + initRoot(60); + assertThat(getConfig().snapshot().periods().size(), is(2)); + assertThat(getConfig().snapshot().periods(0), is(60)); + assertThat(getConfig().snapshot().periods(1), is(300)); + } + + @Test + public void correct_config_is_snooped_default_interval() throws Exception { + String getAdminXmlIntervalNotSpecified = "<admin version='2.0'>" + + " <adminserver hostalias='mockhost' />" + + "</admin>"; + + TestDriver tester = new TestDriver(); + root = tester.buildModel(getAdminXmlIntervalNotSpecified + getContent()); + assertThat(getConfig().snapshot().periods().size(), is(2)); + assertThat(getConfig().snapshot().periods(0), is(60)); + assertThat(getConfig().snapshot().periods(1), is(300)); + } + + @Test(expected = Exception.class) + public void invalid_model_1() throws Exception { + initRoot(120); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java new file mode 100644 index 00000000000..e53c0038421 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.Dispatch; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ClusterTest { + + @Test + public void requireThatContentSearchIsApplied() throws ParseException { + ContentCluster cluster = newContentCluster( + "<search>" + + " <query-timeout>1.1</query-timeout>" + + " <visibility-delay>2.3</visibility-delay>" + + "</search>"); + IndexedSearchCluster searchCluster = cluster.getSearch().getIndexed(); + assertNotNull(searchCluster); + assertEquals(1.1, searchCluster.getQueryTimeout(), 1E-6); + assertEquals(2.3, searchCluster.getVisibilityDelay(), 1E-6); + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + cluster.getSearch().getConfig(builder); + ProtonConfig proton = new ProtonConfig(builder); + assertEquals(searchCluster.getVisibilityDelay(), proton.documentdb(0).visibilitydelay(), 1E-6); + } + + @Test + public void requireThatSearchCoverageIsApplied() throws ParseException { + ContentCluster cluster = newContentCluster( + "<search>" + + " <coverage>" + + " <minimum>0.11</minimum>" + + " <min-wait-after-coverage-factor>0.23</min-wait-after-coverage-factor>" + + " <max-wait-after-coverage-factor>0.58</max-wait-after-coverage-factor>" + + " </coverage>" + + "</search>"); + for (Dispatch tld : cluster.getSearch().getIndexed().getTLDs()) { + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + tld.getConfig(builder); + PartitionsConfig config = new PartitionsConfig(builder); + assertEquals(11.0, config.dataset(0).minimal_searchcoverage(), 1E-6); + assertEquals(0.23, config.dataset(0).higher_coverage_minsearchwait(), 1E-6); + assertEquals(0.58, config.dataset(0).higher_coverage_maxsearchwait(), 1E-6); + } + } + + private static ContentCluster newContentCluster(String contentSearchXml) throws ParseException { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts( + "<hosts>" + + " <host name='localhost'><alias>my_host</alias></host>" + + "</hosts>") + .withServices( + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='my_host' />" + + " </admin>" + + " <content version='1.0'>" + + " <documents>" + + " <document mode='index' type='my_document' />" + + " </documents>" + + " <engine><proton /></engine>" + + " <group>" + + " <node hostalias='my_host' distribution-key='0' />" + + " </group>" + + contentSearchXml + + " </content>" + + "</services>") + .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document")) + .build(); + List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class); + assertEquals(1, contents.size()); + return contents.get(0).getCluster(); + } + + private static SearchDefinition newSearchDefinition(String name) throws ParseException { + SearchBuilder builder = new SearchBuilder(); + builder.importString("search " + name + " { document " + name + " { } }"); + builder.build(); + return new SearchDefinition(name, builder.getSearch(name)); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java new file mode 100644 index 00000000000..e3b0120360e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.content.ContentSearch; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.apache.commons.io.input.CharSequenceInputStream; +import org.junit.Test; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomContentSearchBuilderTest { + + @Test + public void requireThatDefaultsAreNull() throws Exception { + ContentSearch search = newContentSearch( + "<content/>"); + assertNull(search.getVisibilityDelay()); + assertNull(search.getQueryTimeout()); + } + + @Test + public void requireThatEmptySearchIsSafe() throws Exception { + ContentSearch search = newContentSearch( + "<content>" + + " <search/>" + + "</content>"); + assertNull(search.getVisibilityDelay()); + assertNull(search.getQueryTimeout()); + } + + @Test + public void requireThatContentSearchCanBeBuilt() throws Exception { + ContentSearch search = newContentSearch( + "<content>" + + " <search>" + + " <query-timeout>1.1</query-timeout>" + + " <visibility-delay>2.3</visibility-delay>" + + " </search>" + + "</content>"); + assertEquals(1.1, search.getQueryTimeout(), 1E-6); + assertEquals(2.3, search.getVisibilityDelay(), 1E-6); + } + + private static ContentSearch newContentSearch(String xml) throws Exception { + return DomContentSearchBuilder.build( + new ModelElement(DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .getDocumentElement())); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java new file mode 100644 index 00000000000..435382c99b6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.SearchCoverage; +import org.apache.commons.io.input.CharSequenceInputStream; +import org.junit.Test; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomSearchCoverageBuilderTest { + + @Test + public void requireThatDefaultsAreNull() throws Exception { + SearchCoverage coverage = newSearchCoverage( + "<content/>"); + assertNull(coverage.getMinimum()); + assertNull(coverage.getMinWaitAfterCoverageFactor()); + assertNull(coverage.getMaxWaitAfterCoverageFactor()); + } + + @Test + public void requireThatEmptySearchIsSafe() throws Exception { + SearchCoverage coverage = newSearchCoverage( + "<content>" + + " <search/>" + + "</content>"); + assertNull(coverage.getMinimum()); + assertNull(coverage.getMinWaitAfterCoverageFactor()); + assertNull(coverage.getMaxWaitAfterCoverageFactor()); + } + + @Test + public void requireThatEmptyCoverageIsSafe() throws Exception { + SearchCoverage coverage = newSearchCoverage( + "<content>" + + " <search>" + + " <coverage/>" + + " </search>" + + "</content>"); + assertNull(coverage.getMinimum()); + assertNull(coverage.getMinWaitAfterCoverageFactor()); + assertNull(coverage.getMaxWaitAfterCoverageFactor()); + } + + @Test + public void requireThatSearchCoverageCanBeBuilt() throws Exception { + SearchCoverage coverage = newSearchCoverage( + "<content>" + + " <search>" + + " <coverage>" + + " <minimum>0.11</minimum>" + + " <min-wait-after-coverage-factor>0.23</min-wait-after-coverage-factor>" + + " <max-wait-after-coverage-factor>0.58</max-wait-after-coverage-factor>" + + " </coverage>" + + " </search>" + + "</content>"); + assertEquals(0.11, coverage.getMinimum(), 1E-6); + assertEquals(0.23, coverage.getMinWaitAfterCoverageFactor(), 1E-6); + assertEquals(0.58, coverage.getMaxWaitAfterCoverageFactor(), 1E-6); + } + + private static SearchCoverage newSearchCoverage(String xml) throws Exception { + return DomSearchCoverageBuilder.build( + new ModelElement(DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .getDocumentElement())); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java new file mode 100644 index 00000000000..7c31f05908d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.TuningDispatch; +import org.apache.commons.io.input.CharSequenceInputStream; +import org.junit.Test; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomTuningDispatchBuilderTest { + + @Test + public void requireThatDefaultsAreNull() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content/>"); + assertNull(dispatch.getMaxHitsPerPartition()); + } + + @Test + public void requireThatEmptyTuningIsSafe() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content>" + + " <tuning/>" + + "</content>"); + assertNull(dispatch.getMaxHitsPerPartition()); + } + + @Test + public void requireThatEmptydispatchIsSafe() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content>" + + " <tuning>" + + " <dispatch/>" + + " </tuning>" + + "</content>"); + assertNull(dispatch.getMaxHitsPerPartition()); + assertNull(dispatch.getMinGroupCoverage()); + assertNull(dispatch.getMinActiveDocsCoverage()); + assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + } + + @Test + public void requireThatTuningDispatchCanBeBuilt() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content>" + + " <tuning>" + + " <dispatch>" + + " <max-hits-per-partition>69</max-hits-per-partition>" + + " <min-group-coverage>7.5</min-group-coverage>" + + " <min-active-docs-coverage>12.5</min-active-docs-coverage>" + + " </dispatch>" + + " </tuning>" + + "</content>"); + assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); + assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); + assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); + } + @Test + public void requireThatTuningDispatchPolicyRoundRobin() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content>" + + " <tuning>" + + " <dispatch>" + + " <dispatch-policy>round-robin</dispatch-policy>" + + " </dispatch>" + + " </tuning>" + + "</content>"); + assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + } + @Test + public void requireThatTuningDispatchPolicyRandom() throws Exception { + TuningDispatch dispatch = newTuningDispatch( + "<content>" + + " <tuning>" + + " <dispatch>" + + " <dispatch-policy>random</dispatch-policy>" + + " </dispatch>" + + " </tuning>" + + "</content>"); + assertTrue(TuningDispatch.DispatchPolicy.RANDOM == dispatch.getDispatchPolicy()); + } + + private static TuningDispatch newTuningDispatch(String xml) throws Exception { + return DomTuningDispatchBuilder.build( + new ModelElement(DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .getDocumentElement())); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java new file mode 100644 index 00000000000..0e8d3cbbff7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.utils; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class for building an application package with content clusters (used for testing only). + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ApplicationPackageBuilder { + + private List<ContentClusterBuilder> contentClusters = new ArrayList<>(); + private List<String> searchDefinitions = new ArrayList<>(); + + public ApplicationPackageBuilder() { + } + + public ApplicationPackageBuilder addCluster(ContentClusterBuilder contentCluster) { + contentClusters.add(contentCluster); + return this; + } + + public ApplicationPackageBuilder addSearchDefinition(String searchDefinition) { + searchDefinitions.add(searchDefinition); + return this; + } + + public VespaModelCreatorWithMockPkg buildCreator() { + return new VespaModelCreatorWithMockPkg(null, getServices(), searchDefinitions); + } + + private String getServices() { + return "<services version='1.0'>\n" + + contentClusters.stream().map(cluster -> cluster.getXml() + "\n").reduce("", (acc, element) -> acc + element) + + "</services>"; + } + +} + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java new file mode 100644 index 00000000000..580f4f648d6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.utils; + +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import org.w3c.dom.Document; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Class for building a content cluster with indexed search (used for testing only). + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ContentClusterBuilder { + + private String name = "mycluster"; + private int redundancy = 1; + private int searchableCopies = 1; + private List<String> docTypes = Arrays.asList("test"); + private String groupXml = getSimpleGroupXml(); + private Optional<String> dispatchXml = Optional.empty(); + private Optional<Double> protonDiskLimit = Optional.empty(); + private Optional<Double> protonMemoryLimit = Optional.empty(); + + public ContentClusterBuilder() { + } + + public ContentClusterBuilder name(String name) { + this.name = name; + return this; + } + + public ContentClusterBuilder redundancy(int redundancy) { + this.redundancy = redundancy; + return this; + } + + public ContentClusterBuilder searchableCopies(int searchableCopies) { + this.searchableCopies = searchableCopies; + return this; + } + + public ContentClusterBuilder docTypes(List<String> docTypes) { + this.docTypes = docTypes; + return this; + } + + public ContentClusterBuilder groupXml(String groupXml) { + this.groupXml = groupXml; + return this; + } + + public ContentClusterBuilder dispatchXml(Optional<String> dispatchXml) { + this.dispatchXml = dispatchXml; + return this; + } + + public ContentClusterBuilder protonDiskLimit(double diskLimit) { + protonDiskLimit = Optional.of(diskLimit); + return this; + } + + public ContentClusterBuilder protonMemoryLimit(double memoryLimit) { + protonMemoryLimit = Optional.of(memoryLimit); + return this; + } + + public ContentCluster build(MockRoot root) { + Document doc = XML.getDocument(getXml()); + return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement()); + } + + public String getXml() { + String xml = "<content version='1.0' id='" + name + "'>\n" + + " <redundancy>" + redundancy + "</redundancy>\n" + + " <documents>\n" + + docTypes.stream().map(type -> " <document mode='index' type='" + type + "'/>\n").collect(Collectors.joining("\n")) + + " </documents>\n" + + " <engine>\n" + + " <proton>\n" + + " <searchable-copies>" + searchableCopies + "</searchable-copies>\n" + + getResourceLimitsXml(" ") + + " </proton>\n" + + " </engine>\n"; + if (dispatchXml.isPresent()) { + xml += dispatchXml.get(); + } + return xml + groupXml + + "</content>"; + } + + private static String getSimpleGroupXml() { + return " <group>\n" + + " <node distribution-key='0' hostalias='mockhost'/>\n" + + " </group>\n"; + } + + private String getResourceLimitsXml(String indent) { + if (protonDiskLimit.isPresent() || protonMemoryLimit.isPresent()) { + String xml = indent + "<resource-limits>\n" + + getXmlLine("disk", protonDiskLimit, indent + " ") + + getXmlLine("memory", protonMemoryLimit, indent + " ") + + indent + "</resource-limits>\n"; + return xml; + } + return ""; + } + + private static String getXmlLine(String tag, Optional<Double> value, String indent) { + if (value.isPresent()) { + return indent + "<" + tag + ">" + value.get() + "</" + tag + ">\n"; + } + return ""; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java new file mode 100644 index 00000000000..e64a39198b4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.utils; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import org.w3c.dom.Document; + +import java.util.List; +import java.util.Optional; + +/** + * For testing purposes only. + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ContentClusterUtils { + + public static MockRoot createMockRoot(String[] hosts) throws Exception { + return createMockRoot(hosts, ApplicationPackageUtils.generateSearchDefinition("test")); + } + + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) { + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .modelHostProvisioner(provisioner) + .build(); + return new MockRoot("", deployState); + + } + public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) throws Exception { + return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions); + } + + public static MockRoot createMockRoot(List<String> searchDefinitions) { + return createMockRoot(new SingleNodeProvisioner(), searchDefinitions); + } + + public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception { + Document doc = XML.getDocument(clusterXml); + return new ContentCluster.Builder(null, null).build(root, doc.getDocumentElement()); + } + + public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception { + MockRoot root = createMockRoot(searchDefinitions); + ContentCluster cluster = createCluster(clusterXml, root); + root.freezeModelTopology(); + cluster.validate(); + return cluster; + } + + public static ContentCluster createCluster(String clusterXml) throws Exception { + return createCluster(clusterXml, ApplicationPackageUtils.generateSearchDefinitions("test")); + } + + public static String createClusterXml(String groupXml, int redundancy, int searchableCopies) { + return createClusterXml(groupXml, Optional.empty(), redundancy, searchableCopies); + } + + public static String createClusterXml(String groupXml, Optional<String> dispatchXml, int redundancy, int searchableCopies) { + return new ContentClusterBuilder(). + groupXml(groupXml). + dispatchXml(dispatchXml). + redundancy(redundancy). + searchableCopies(searchableCopies).getXml(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java new file mode 100644 index 00000000000..a0817bf7446 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.utils; + +/** + * Class for building a search definition (used for testing only). + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class SearchDefinitionBuilder { + + private String name = "test"; + private String content = ""; + + public SearchDefinitionBuilder() { + } + + public SearchDefinitionBuilder name(String name) { + this.name = name; + return this; + } + + public SearchDefinitionBuilder content(String content) { + this.content = content; + return this; + } + + public String build() { + return "search " + name + " {\n" + + " document " + name + " {\n" + + content + "\n" + + " }\n" + + "}"; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java new file mode 100644 index 00000000000..cb8b3e4bc02 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.test.MockHosts; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class FileDistributorTestCase { + @Test + public void fileDistributor() { + MockHosts hosts = new MockHosts(); + + FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry()); + + FileReference ref1 = fileDistributor.sendFileToHosts("components/path1", Arrays.asList(hosts.host1, hosts.host2)); + FileReference ref2 = fileDistributor.sendFileToHosts("path2", Arrays.asList(hosts.host3)); + + assertEquals(new HashSet<>(Arrays.asList(hosts.host1, hosts.host2, hosts.host3)), + fileDistributor.getTargetHosts()); + + assertTrue( ref1 != null ); + assertTrue( ref2 != null ); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java new file mode 100644 index 00000000000..c7c1ae6784f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class GenericServicesModelTest { + + @Test + public void test_generic_services_builder() { + GenericServicesBuilder builder = new GenericServicesBuilder(); + assertThat(builder.handlesElements().size(), is(1)); + assertThat(builder.handlesElements().get(0), is(ConfigModelId.fromName("service"))); + } + + @Test + public void test_generic_services_model() { + MockRoot root = new MockRoot(); + GenericServicesModel model = new GenericServicesModel(ConfigModelContext.createFromParentAndId(null, root, "foo")); + assertThat(model.serviceClusters().size(), is(0)); + model.addCluster(new ServiceCluster(root, "mycluster", "/bin/foo")); + assertThat(model.serviceClusters().size(), is(1)); + assertThat(model.serviceClusters().get(0).getName(), is("mycluster")); + } + + @Test + public void test_generic_services_parsing() throws IOException, SAXException { + final String hosts = + "<hosts>" + + "<host name=\"localhost\">" + + " <alias>mockhost</alias>" + + " </host> " + + "</hosts>"; + String services = "<services version=\"1.0\">" + + "<service id=\"me\" name=\"foo\" command=\"/bin/bar\" version=\"1.0\">" + + "<node hostalias=\"mockhost\" />" + + "</service>" + + "</services>"; + VespaModel model = new VespaModel(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build()); + GenericServicesModel gsModel = (GenericServicesModel) model.configModelRepo().get("me"); + assertThat(gsModel.serviceClusters().size(), is(1)); + assertThat(gsModel.serviceClusters().get(0).getName(), is("foo")); + assertThat(gsModel.serviceClusters().get(0).services().size(), is(1)); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java new file mode 100644 index 00000000000..05c007b155b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.config.GenericConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.generic.service.Service; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import org.junit.BeforeClass; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Tests that generic services result in correct sentinel config settings + * + * @author vegardh + */ +public class GenericServicesTest { + + private static VespaModel model; + + @BeforeClass + public static void getModel() throws IOException, SAXException { + String appDir = "src/test/cfg/application/app_genericservices"; + ApplicationPackage app = FilesApplicationPackage.fromFile(new File(appDir)); + model = new VespaModel(app); + } + + @Test + public void testServicesSentinelConfig() throws IOException, SAXException { + String sentinelConfigId1="hosts/bogusname1/sentinel"; + String sentinelConfigId2="hosts/bogusname2/sentinel"; + String sentinelConfigId3="hosts/bogusname3/sentinel"; + String sentinelConfigId4="hosts/bogusname4/sentinel"; + SentinelConfig sentinel1 = model.getConfig(SentinelConfig.class, sentinelConfigId1); + SentinelConfig sentinel2 = model.getConfig(SentinelConfig.class, sentinelConfigId2); + SentinelConfig sentinel3 = model.getConfig(SentinelConfig.class, sentinelConfigId3); + SentinelConfig sentinel4 = model.getConfig(SentinelConfig.class, sentinelConfigId4); + + assertServiceExists(sentinel1, "myservice", "mycmd1.sh", "myservice/0", true, true); + assertServiceExists(sentinel2, "myservice", "mycmd1.sh", "myservice/1", true, true); + assertServiceExists(sentinel3, "myservice", "mycmd1.sh", "myservice/2", true, true); + assertServiceExists(sentinel3, "myservice2", "mycmd1.sh", "myservice/3", true, true); + assertServiceExists(sentinel3, "myotherservice", "/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR", "myotherservice/0", true, true); + assertServiceExists(sentinel4, "myotherservice", "/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR", "myotherservice/1", true, true); + } + + private void assertServiceExists(SentinelConfig sentinel, String serviceName, String cmd, String configId, boolean autostart, boolean autorestart) { + boolean matches = false; + Iterator<SentinelConfig.Service> it = sentinel.service().iterator(); + while (!matches && it.hasNext()) { + SentinelConfig.Service service = it.next(); + matches = service.autorestart() == autorestart && + service.autostart() == autostart && + service.name().equals(serviceName) && + service.id().equals(configId) && + service.command().equals(cmd); + } + assertTrue(matches); + } + + @Test + public void testServicesModel() throws IOException, SAXException { + // Testing that this model can be constructed only for now + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java new file mode 100755 index 00000000000..fba740567f8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -0,0 +1,203 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.routing.test; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RoutingTestCase { + + private static final boolean WRITE_FILES = false; + + @Test + @Ignore // TODO: Why? + public void testRoutingContent() throws IOException { + assertApplication(new File("src/test/cfg/routing/contentsimpleconfig")); + assertApplication(new File("src/test/cfg/routing/content_two_clusters")); + } + + @Test + public void testRouting() throws IOException { + assertApplication(new File("src/test/cfg/routing/unexpectedrecipient")); + assertApplication(new File("src/test/cfg/routing/servicenotfound")); + assertApplication(new File("src/test/cfg/routing/routenotfoundinroute")); + assertApplication(new File("src/test/cfg/routing/routenotfound")); + assertApplication(new File("src/test/cfg/routing/routeconfig")); + assertApplication(new File("src/test/cfg/routing/replaceroute")); + assertApplication(new File("src/test/cfg/routing/replacehop")); + assertApplication(new File("src/test/cfg/routing/mismatchedrecipient")); + assertApplication(new File("src/test/cfg/routing/hopnotfound")); + assertApplication(new File("src/test/cfg/routing/hoperrorinroute")); + assertApplication(new File("src/test/cfg/routing/hoperrorinrecipient")); + assertApplication(new File("src/test/cfg/routing/hoperror")); + assertApplication(new File("src/test/cfg/routing/hopconfig")); + assertApplication(new File("src/test/cfg/routing/emptyroute")); + assertApplication(new File("src/test/cfg/routing/emptyhop")); + assertApplication(new File("src/test/cfg/routing/duplicateroute")); + assertApplication(new File("src/test/cfg/routing/duplicatehop")); + assertApplication(new File("src/test/cfg/routing/defaultconfig")); + } + + /** + * Tests whether or not the given application produces the expected output. When creating new tests, create an + * application directory containing the necessary setup files, and call this method with a TRUE create flag. + * + * @param application The application directory. + * @throws IOException + */ + private static void assertApplication(File application) throws IOException { + assertTrue(application.isDirectory()); + String applicationName = application.getName(); + System.out.println("Testing route configuration from application '" + applicationName + "'.."); + Map<String, File> files = new HashMap<>(); + for (File file : application.listFiles(new ContentFilter())) { + files.put(file.getName(), file); + } + String path = null; + try { + path = application.getCanonicalPath(); + } catch (IOException e) { + fail("Could not resolve path for application '" + applicationName + "'."); + } + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg(path); + + VespaModel model = creator.create(); + List<String> errors = model.getRouting().getErrors(); + if (errors.isEmpty()) { + if (files.containsKey("errors.txt")) { + if (WRITE_FILES) { + files.remove("errors.txt").delete(); + } else { + fail("Route verification did not fail."); + } + } + MessagebusConfig.Builder mBusB = new MessagebusConfig.Builder(); + model.getConfig(mBusB, ""); + MessagebusConfig mBus = new MessagebusConfig(mBusB); + assertConfigFileContains(application, files, "messagebus.cfg", mBus); + + DocumentrouteselectorpolicyConfig.Builder drB = new DocumentrouteselectorpolicyConfig.Builder(); + model.getConfig(drB, ""); + DocumentrouteselectorpolicyConfig dr = new DocumentrouteselectorpolicyConfig(drB); + assertConfigFileContains(application, files, "documentrouteselectorpolicy.cfg", dr); + } else { + StringBuilder msg = new StringBuilder(); + for (String error : errors) { + msg.append(error).append("\n"); + } + assertFileContains(application, files, "errors.txt", msg.toString()); + } + System.out.println("\tDone."); + } + + /** + * Tests whether or not a given file exists and contains some expected content. + * + * @param application The application directory. + * @param files The filtered list of files within the application. + * @param fileName The name of the file whose content to check. + * @param expectedContent The content required in the file being checked. + */ + private static void assertFileContains(File application, Map<String, File> files, + String fileName, String expectedContent) { + if (WRITE_FILES) { + files.put(fileName, writeFile(application, fileName, expectedContent.trim() + "\n")); + } + if (!files.containsKey(fileName)) { + fail("Expected file '" + fileName + "' not found.\nExpected content: " + expectedContent); + return; + } + System.out.println("\tVerifying content of '" + fileName + "'."); + StringBuilder content = new StringBuilder(); + + try { + BufferedReader reader = new BufferedReader(new FileReader(files.get(fileName))); + String line = reader.readLine(); + while (line != null) { + content.append(line).append("\n"); + line = reader.readLine(); + } + reader.close(); + } catch (FileNotFoundException e) { + fail("File '" + fileName + "' not found."); + } catch (IOException e) { + fail("Failed to read content of file '" + fileName + "."); + } + + assertEquals(content.toString().trim(), expectedContent.replace("\r", "").trim()); + } + /** + * Tests whether or not a given file exists and contains some expected content. + * + * @param application The application directory. + * @param files The filtered list of files within the application. + * @param fileName The name of the file whose content to check. + * @param config The config required in the file being checked. + * @throws IOException + */ + private static void assertConfigFileContains(File application, Map<String, File> files, + String fileName, ConfigInstance config) throws IOException { + final String configString = config.toString(); + if (WRITE_FILES) { + files.put(fileName, writeFile(application, fileName, configString.trim() + "\n")); + } + if (!files.containsKey(fileName)) { + fail("Expected file '" + fileName + "' not found."); + return; + } + System.out.println("\tVerifying content of '" + fileName + "'."); + assertSerializedConfigEquals(IOUtils.readFile(files.get(fileName)), configString); + } + + /** + * Writes content to a specific file. + * + * @param application The application directory. + * @param name The name of the file to write. + * @param content The content to write. + * @return The file written. + */ + private static File writeFile(File application, String name, String content) { + File ret = null; + try { + name = application.getCanonicalPath() + "/" + name; + System.out.println("\tWriting file '" + name + "'."); + + PrintWriter writer = new PrintWriter(new FileWriter(name)); + writer.print(content); + writer.close(); + + ret = new File(name); + } catch (IOException e) { + fail(e.getMessage()); + } + return ret; + } + + /** + * Helper class to filter what files within an application directory gets added to the files index. + */ + private static class ContentFilter implements FilenameFilter { + public boolean accept(File file, String name) { + return !name.equals(".git") && !name.equals(".svn"); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java new file mode 100644 index 00000000000..135d5302aee --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java @@ -0,0 +1,382 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.content.DispatchSpec; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.search.utils.DispatchUtils; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; +import static com.yahoo.vespa.model.search.utils.DispatchUtils.getDataset; +import static com.yahoo.vespa.model.search.utils.DispatchUtils.getFdispatchrcConfig; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.hamcrest.Matchers.containsString; + +/** + * Unit tests for multi-level dispatchers in an indexed content cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class MultilevelDispatchTest { + + private static class EngineAsserter { + private List<PartitionsConfig.Dataset.Engine> engines; + private int engineIdx = 0; + public EngineAsserter(int numParts, int numEngines, Dispatch dispatch) { + PartitionsConfig.Dataset dataset = getDataset(dispatch); + assertEquals(numParts, dataset.numparts()); + assertEquals(PartitionsConfig.Dataset.Querydistribution.AUTOMATIC, dataset.querydistribution()); + engines = dataset.engine(); + assertEquals(numEngines, engines.size()); + } + EngineAsserter assertEngine(int rowId, int partitionId, String connectSpec) { + DispatchUtils.assertEngine(rowId, partitionId, connectSpec, engines.get(engineIdx++)); + return this; + } + } + + private String getGroupXml() { + return " <group>\n" + + " <node distribution-key='10' hostalias='mh0'/>\n" + + " <node distribution-key='11' hostalias='mh1'/>\n" + + " <node distribution-key='12' hostalias='mh2'/>\n" + + " <node distribution-key='13' hostalias='mh3'/>\n" + + " <node distribution-key='14' hostalias='mh4'/>\n" + + " <node distribution-key='15' hostalias='mh5'/>\n" + + " </group>\n"; + } + + private String getSimpleDispatchXml() { + return " <dispatch>\n" + + " <num-dispatch-groups>2</num-dispatch-groups>\n" + + " </dispatch>\n"; + } + + private String getDispatchXml() { + return " <dispatch>\n" + + " <group>\n" + + " <node distribution-key='10'/>\n" + + " <node distribution-key='12'/>\n" + + " <node distribution-key='14'/>\n" + + " </group>\n" + + " <group>\n" + + " <node distribution-key='11'/>\n" + + " <node distribution-key='13'/>\n" + + " <node distribution-key='15'/>\n" + + " </group>\n" + + " </dispatch>\n"; + } + + private ContentCluster createCluster(String dispatchXml) throws Exception { + String[] hosts = {"mh0", "mh1", "mh2", "mh3", "mh4", "mh5"}; + MockRoot root = ContentClusterUtils.createMockRoot(hosts); + ContentCluster cluster = ContentClusterUtils.createCluster(createClusterXml(getGroupXml(), Optional.of(dispatchXml), 1, 1), root); + + AbstractConfigProducer<Dispatch> dispatchParent = new SimpleConfigProducer<>(root, "tlds"); + HostResource hostResource = new HostResource(new Host(root, "mockhost")); + IndexedSearchCluster index = cluster.getSearch().getIndexed(); + index.addTld(dispatchParent, hostResource); + index.setupDispatchGroups(); + + root.freezeModelTopology(); + cluster.validate(); + return cluster; + } + + private List<Dispatch> getDispatchers(Dispatch tld) { + DispatchGroup group = tld.getDispatchGroup(); + List<Dispatch> dispatchers = new ArrayList<>(); + for (SearchInterface dispatch : group.getSearchersIterable()) { + dispatchers.add((Dispatch)dispatch); + } + return dispatchers; + } + + private void assertDispatchAndSearchNodes(int partId, Dispatch[] dispatchers, String[] connectSpecs, SearchNode[] searchNodes) { + assertEquals(dispatchers.length, connectSpecs.length); + assertEquals(connectSpecs.length, searchNodes.length); + int searchNodeIdx = 0; + for (int rowId = 0; rowId < dispatchers.length; ++rowId) { + assertDispatchAndSearchNodes(rowId, partId, searchNodes[searchNodeIdx++].getDistributionKey(), + dispatchers[rowId], connectSpecs, searchNodes); + } + } + + private void assertDispatchAndSearchNodes(int expRowId, int expPartId, int expDistributionKey, Dispatch dispatch, String[] connectSpecs, SearchNode[] searchNodes) { + assertEquals(expRowId, dispatch.getNodeSpec().rowId()); + assertEquals(expPartId, dispatch.getNodeSpec().partitionId()); + assertEquals("mycluster/search/cluster.mycluster/dispatchers/dispatch." + expDistributionKey, dispatch.getConfigId()); + assertEquals(expPartId, getFdispatchrcConfig(dispatch).partition()); + assertEquals(1, getFdispatchrcConfig(dispatch).dispatchlevel()); + + int numEngines = connectSpecs.length; + EngineAsserter ea = new EngineAsserter(numEngines, numEngines, dispatch); + for (int i = 0; i < numEngines; ++i) { + ea.assertEngine(0, i, connectSpecs[i]); + assertEquals(i, searchNodes[i].getNodeSpec().partitionId()); + } + } + + @Test + public void requireThatDispatchGroupsCanBeAutomaticallySetup() throws Exception { + ContentCluster cr = createCluster(getSimpleDispatchXml()); + IndexedSearchCluster ix = cr.getSearch().getIndexed(); + Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); + + assertEquals("tlds/tld.0", tld.getConfigId()); + assertEquals(0, getFdispatchrcConfig(tld).dispatchlevel()); + new EngineAsserter(2, 6, tld). + assertEngine(0, 0, "tcp/mh0:19113"). + assertEngine(1, 0, "tcp/mh1:19113"). + assertEngine(2, 0, "tcp/mh2:19113"). + assertEngine(0, 1, "tcp/mh3:19113"). + assertEngine(1, 1, "tcp/mh4:19113"). + assertEngine(2, 1, "tcp/mh5:19113"); + + List<Dispatch> ds = getDispatchers(tld); + assertEquals(6, ds.size()); + { // dispatch group 1 + Dispatch[] dispatchers = {ds.get(0), ds.get(1), ds.get(2)}; + String[] specs = {"tcp/mh0:19104", "tcp/mh1:19104", "tcp/mh2:19104"}; + SearchNode[] searchNodes = {ix.getSearchNode(0), ix.getSearchNode(1), ix.getSearchNode(2)}; + assertDispatchAndSearchNodes(0, dispatchers, specs, searchNodes); + } + { // dispatch group 2 + Dispatch[] dispatchers = {ds.get(3), ds.get(4), ds.get(5)}; + String[] specs = {"tcp/mh3:19104", "tcp/mh4:19104", "tcp/mh5:19104"}; + SearchNode[] searchNodes = {ix.getSearchNode(3), ix.getSearchNode(4), ix.getSearchNode(5)}; + assertDispatchAndSearchNodes(1, dispatchers, specs, searchNodes); + } + } + + @Test + public void requireThatMaxHitsIsScaled() throws Exception { + ContentCluster cr = createCluster(getSimpleDispatchXml() + getMaxhitsTuning()); + IndexedSearchCluster ix = cr.getSearch().getIndexed(); + Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + tld.getConfig(builder); + PartitionsConfig config = new PartitionsConfig(builder); + assertThat(config.dataset().size(), is(1)); + assertThat(config.dataset(0).maxhitspernode(), is(300)); + for (Dispatch dispatch : getDispatchers(tld)) { + PartitionsConfig.Builder b = new PartitionsConfig.Builder(); + dispatch.getConfig(b); + PartitionsConfig c= new PartitionsConfig(b); + assertThat(c.dataset().size(), is(1)); + assertThat(c.dataset(0).maxhitspernode(), is(100)); + } + } + + private String getMaxhitsTuning() { + return "<tuning>" + + " <dispatch>" + + " <max-hits-per-partition>100</max-hits-per-partition>" + + " </dispatch>" + + "</tuning>"; + } + + + @Test + public void requireThatSearchCoverageIsSetInMultilevelSetup() throws Exception { + ContentCluster cr = createCluster(getSimpleDispatchXml() + getCoverage()); + Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + tld.getConfig(builder); + PartitionsConfig config = new PartitionsConfig(builder); + assertThat(config.dataset().size(), is(1)); + assertEquals(95.0, config.dataset(0).minimal_searchcoverage(), 0.1); + for (Dispatch dispatch : getDispatchers(tld)) { + PartitionsConfig.Builder b = new PartitionsConfig.Builder(); + dispatch.getConfig(b); + PartitionsConfig c= new PartitionsConfig(b); + assertThat(c.dataset().size(), is(1)); + assertEquals(95.0, c.dataset(0).minimal_searchcoverage(), 0.1); + } + } + + @Test + public void requireThatSearchCoverageIsSetInSingleLevelSetup() throws Exception { + TestRoot root = new TestDriver(true).buildModel(new MockApplicationPackage.Builder() + .withServices("<services version='1.0'>" + + "<content id='stateful' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <documents><document mode='index' type='music' /></documents>" + + " <nodes>" + + " <node distribution-key='1' hostalias='mockroot' />" + + " </nodes>" + + " <search><coverage><minimum>0.95</minimum></coverage></search>" + + "</content>" + + "<jdisc id='foo' version='1.0'>" + + " <search />" + + " <nodes><node hostalias='mockroot' /></nodes>" + + "</jdisc>" + + "</services>") + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()); + PartitionsConfig config = root.getConfig(PartitionsConfig.class, "stateful/search/cluster.stateful/tlds/foo.0.tld.0"); + assertThat(config.dataset().size(), is(1)); + assertEquals(95.0, config.dataset(0).minimal_searchcoverage(), 0.1); + } + + private String getCoverage() { + return "<search>" + + " <coverage>" + + " <minimum>0.95</minimum>" + + " </coverage>" + + "</search>"; + } + + @Test + public void requireThatDispatchGroupsCanBeExplicitlySpecified() throws Exception { + ContentCluster cr = createCluster(getDispatchXml()); + IndexedSearchCluster ix = cr.getSearch().getIndexed(); + Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); + + assertEquals("tlds/tld.0", tld.getConfigId()); + assertEquals(0, getFdispatchrcConfig(tld).dispatchlevel()); + new EngineAsserter(2, 6, tld). + assertEngine(0, 0, "tcp/mh0:19113"). + assertEngine(1, 0, "tcp/mh2:19113"). + assertEngine(2, 0, "tcp/mh4:19113"). + assertEngine(0, 1, "tcp/mh1:19113"). + assertEngine(1, 1, "tcp/mh3:19113"). + assertEngine(2, 1, "tcp/mh5:19113"); + + List<Dispatch> ds = getDispatchers(tld); + assertEquals(6, ds.size()); + { // dispatch group 1 + Dispatch[] dispatchers = {ds.get(0), ds.get(1), ds.get(2)}; + String[] specs = {"tcp/mh0:19104", "tcp/mh2:19104", "tcp/mh4:19104"}; + SearchNode[] searchNodes = {ix.getSearchNode(0), ix.getSearchNode(2), ix.getSearchNode(4)}; + assertDispatchAndSearchNodes(0, dispatchers, specs, searchNodes); + } + { // dispatch group 2 + Dispatch[] dispatchers = {ds.get(3), ds.get(4), ds.get(5)}; + String[] specs = {"tcp/mh1:19104", "tcp/mh3:19104", "tcp/mh5:19104"}; + SearchNode[] searchNodes = {ix.getSearchNode(1), ix.getSearchNode(3), ix.getSearchNode(5)}; + assertDispatchAndSearchNodes(1, dispatchers, specs, searchNodes); + } + } + + @Test + public void requireThatUnevenDispatchGroupsCanBeCreated() { + List<SearchNode> searchNodes = createSearchNodes(5); + List<DispatchSpec.Group> groups = DispatchGroupBuilder.createDispatchGroups(searchNodes, 3); + assertEquals(3, groups.size()); + assertGroup(new int[]{0, 1}, groups.get(0)); + assertGroup(new int[]{2, 3}, groups.get(1)); + assertGroup(new int[]{4}, groups.get(2)); + } + + private List<SearchNode> createSearchNodes(int numNodes) { + List<SearchNode> searchNodes = new ArrayList<>(); + MockRoot root = new MockRoot(""); + for (int i = 0; i < numNodes; ++i) { + searchNodes.add(SearchNode.create(root, "mynode" + i, i, new NodeSpec(0, i), "mycluster", null, false)); + } + return searchNodes; + } + + private void assertGroup(int[] nodes, DispatchSpec.Group group) { + assertEquals(nodes.length, group.getNodes().size()); + for (int i = 0; i < nodes.length; ++i) { + assertEquals(nodes[i], group.getNodes().get(i).getDistributionKey()); + } + } + + private ContentCluster createIllegalSetupWithMultipleNodeReferences() throws Exception { + String dispatchXml = " <dispatch>\n" + + " <group>\n" + + " <node distribution-key='10'/>\n" + + " <node distribution-key='11'/>\n" + + " <node distribution-key='12'/>\n" + + " </group>\n" + + " <group>\n" + + " <node distribution-key='12'/>\n" + + " <node distribution-key='13'/>\n" + + " <node distribution-key='14'/>\n" + + " </group>\n" + + " </dispatch>\n"; + return createCluster(dispatchXml); + } + + private ContentCluster createIllegalSetupWithMissingNodeReferences() throws Exception { + String dispatchXml = " <dispatch>\n" + + " <group>\n" + + " <node distribution-key='10'/>\n" + + " <node distribution-key='11'/>\n" + + " </group>\n" + + " <group>\n" + + " <node distribution-key='13'/>\n" + + " <node distribution-key='14'/>\n" + + " </group>\n" + + " </dispatch>\n"; + return createCluster(dispatchXml); + } + + private ContentCluster createIllegalSetupWithIllegalNodeReference() throws Exception { + String dispatchXml = " <dispatch>\n" + + " <group>\n" + + " <node distribution-key='10'/>\n" + + " <node distribution-key='11'/>\n" + + " <node distribution-key='12'/>\n" + + " </group>\n" + + " <group>\n" + + " <node distribution-key='13'/>\n" + + " <node distribution-key='14'/>\n" + + " <node distribution-key='15'/>\n" + + " <node distribution-key='19'/>\n" + + " </group>\n" + + " </dispatch>\n"; + return createCluster(dispatchXml); + } + + @Test + public void requireThatWeReferenceNodesOnlyOnceWhenSettingUpDispatchGroups() { + try { + createIllegalSetupWithMultipleNodeReferences(); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("node with distribution key '12' is referenced multiple times")); + } + } + + @Test + public void requireThatWeReferenceAllNodesWhenSettingUpDispatchGroups() { + try { + createIllegalSetupWithMissingNodeReferences(); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("2 node(s) with distribution keys [12, 15] are not referenced")); + } + } + + @Test + public void requireThatWeReferenceValidNodesWhenSettingUpDispatchGroups() throws Exception { + try { + createIllegalSetupWithIllegalNodeReference(); + assertFalse("Did not get expected Exception", true); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("node with distribution key '19' does not exists")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java new file mode 100644 index 00000000000..e3f4558076c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class TldTest { + + @Test + public void requireThatServicesIsParsed() { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts("<hosts><host name='localhost'><alias>mockhost</alias></host><host name='my.other.host'><alias>mockhost2</alias></host></hosts>") + .withServices( + "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='mockhost' />" + + " </admin>" + + " <jdisc version='1.0' id='default'>" + + " <search />" + + " <nodes>" + + " <node hostalias='mockhost'/>" + + " </nodes>" + + " </jdisc>" + + " <content version='1.0' id='foo'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group>" + + " <node hostalias='mockhost' distribution-key='0'/>" + + " <node hostalias='mockhost2' distribution-key='1'/>" + + " </group>" + + " <tuning>" + + " <dispatch>" + + " <max-hits-per-partition>69</max-hits-per-partition>" + + " <use-local-node>true</use-local-node>" + + " </dispatch>" + + " </tuning>" + + " </content>" + + "</services>") + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build(); + + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); + PartitionsConfig config = new PartitionsConfig(builder); + + assertEquals(1, config.dataset().size()); + assertEquals(69, config.dataset(0).maxhitspernode()); + assertEquals(1, config.dataset(0).engine().size()); + } + + @Test + public void requireThatUseLocalPolicyIsOk() { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts( + "<hosts>" + + "<host name='search.node1'><alias>search1</alias></host>" + + "<host name='search.node2'><alias>search2</alias></host>" + + "<host name='jdisc.host.other'><alias>gateway</alias></host>" + + "</hosts>") + .withServices( + "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='gateway' />" + + " </admin>" + + " <jdisc version='1.0' id='default'>" + + " <search />" + + " <nodes>" + + " <node hostalias='search1'/>" + + " <node hostalias='search2'/>" + + " </nodes>" + + " </jdisc>" + + " <jdisc version='1.0' id='gw'>" + + " <document-api/>" + + " <nodes>" + + " <node hostalias='gateway'/>" + + " </nodes>" + + " </jdisc>" + + " <content version='1.0' id='foo'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <group name='topGroup'>" + + " <distribution partitions='1|*'/>" + + " <group name='group1' distribution-key='0'>" + + " <node hostalias='search1' distribution-key='0'/>" + + " </group>" + + " <group name='group2' distribution-key='1'>" + + " <node hostalias='search2' distribution-key='1'/>" + + " </group>" + + " </group>" + + " <tuning>" + + " <dispatch>" + + " <use-local-node>true</use-local-node>" + + " </dispatch>" + + " </tuning>" + + " </content>" + + "</services>") + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build(); + + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/gw.0.tld.0"); + PartitionsConfig config = new PartitionsConfig(builder); + + assertEquals(1, config.dataset().size()); + //gateway TLD with no local search node gets all search nodes + assertEquals(2, config.dataset(0).engine().size()); + + assertEquals("rowid not equal 0",0,config.dataset(0).engine(0).rowid()); //Load Balance row 0 + assertEquals("partid not equal 0",0,config.dataset(0).engine(0).partid()); + assertTrue("Not configured with correct search node",config.dataset(0).engine(0).name_and_port().contains("search.node1")); + + assertEquals("rowid not equal to 1",1,config.dataset(0).engine(1).rowid()); //Load Balance row 1 + assertEquals("partid no equal to 0",0,config.dataset(0).engine(1).partid()); + assertTrue("Not configured with correct search node",config.dataset(0).engine(1).name_and_port().contains("search.node2")); + + //First container with a local search node + builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); + config = new PartitionsConfig(builder); + + assertEquals(1, config.dataset().size()); + assertEquals(1, config.dataset(0).engine().size()); + assertEquals(0,config.dataset(0).engine(0).rowid()); + assertEquals(0,config.dataset(0).engine(0).partid()); + assertTrue("Not configured with local search node as engine",config.dataset(0).engine(0).name_and_port().contains("search.node1")); + + //Second container with a local search node + builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.1.tld.1"); + config = new PartitionsConfig(builder); + + assertEquals(1, config.dataset().size()); + assertEquals(1, config.dataset(0).engine().size()); + assertEquals(0,config.dataset(0).engine(0).rowid()); + assertEquals(0,config.dataset(0).engine(0).partid()); + assertTrue("Not configured with local search node as engine",config.dataset(0).engine(0).name_and_port().contains("search.node2")); + + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java new file mode 100644 index 00000000000..585c6fe0fb9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.vespa.config.search.IndexschemaConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Ignore; +import org.junit.Test; +import org.xml.sax.SAXException; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +// TODO: Author! +public class DocumentDatabaseTestCase { + + private String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts> " + + " <host name='foo'>" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + private String createVespaServices(List<String> sdNames, String selection, String mode) { + StringBuilder retval = new StringBuilder(); + retval.append("" + + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + "<admin version='2.0'>\n" + + " <adminserver hostalias='node0' />\n" + + "</admin>\n" + + "<container version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " <search/>\n" + + "</container>\n" + + "<content version='1.0' id='test'>\n" + + " <redundancy>1</redundancy>\n"); + retval.append(" <documents>\n"); + for (String sdName : sdNames) { + retval.append("").append(" <document type='").append(sdName).append("' mode='" + mode + "'"); + if (selection != null) + retval.append(" selection='").append(selection).append("'"); + retval.append("/>\n"); + } + retval.append(" </documents>\n"); + retval.append("" + + " <nodes>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </nodes>\n" + + "</content>\n" + + "</services>\n"); + return retval.toString(); + } + + private ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + cluster.getConfig(pb); + return new ProtonConfig(pb); + } + + @Test + public void requireThatWeCanHaveOneSearchDefinition() throws IOException, SAXException, ParseException { + final List<String> sds = Arrays.asList("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); + String type1Id = "test/search/cluster.test/type1"; + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(1, proton.documentdb().size()); + assertEquals("type1", proton.documentdb(0).inputdoctypename()); + assertEquals(type1Id, proton.documentdb(0).configid()); + ProtonConfig nodeCfg = getProtonCfg(contentSearchCluster); + assertEquals(1, nodeCfg.documentdb().size()); + assertEquals("type1", nodeCfg.documentdb(0).inputdoctypename()); + assertEquals(type1Id, nodeCfg.documentdb(0).configid()); + } + + private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) { + IndexschemaConfig icfg = model.getConfig(IndexschemaConfig.class, configId); + assertEquals(1, icfg.indexfield().size()); + assertEquals(indexField, icfg.indexfield(0).name()); + AttributesConfig acfg = model.getConfig(AttributesConfig.class, configId); + assertEquals(1, acfg.attribute().size()); + assertEquals(attributeField, acfg.attribute(0).name()); + RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId); + assertEquals(6, rcfg.rankprofile().size()); + } + + @Test + public void requireThatWeCanHaveMultipleSearchDefinitions() throws IOException, SAXException, ParseException { + final List<String> sds = Arrays.asList("type1", "type2", "type3"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + String type1Id = "test/search/cluster.test/type1"; + String type2Id = "test/search/cluster.test/type2"; + String type3Id = "test/search/cluster.test/type3"; + { + assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(3, proton.documentdb().size()); + assertEquals("type1", proton.documentdb(0).inputdoctypename()); + assertEquals(type1Id, proton.documentdb(0).configid()); + assertEquals("type2", proton.documentdb(1).inputdoctypename()); + assertEquals(type2Id, proton.documentdb(1).configid()); + assertEquals("type3", proton.documentdb(2).inputdoctypename()); + assertEquals(type3Id, proton.documentdb(2).configid()); + } + assertDocTypeConfig(model, type1Id, "f1", "f2"); + assertDocTypeConfig(model, type2Id, "f3", "f4"); + assertDocTypeConfig(model, type3Id, "f5", "f6"); + { + IndexInfoConfig iicfg = model.getConfig(IndexInfoConfig.class, "test/search/cluster.test"); + assertEquals(3, iicfg.indexinfo().size()); + assertEquals("type1", iicfg.indexinfo().get(0).name()); + assertEquals("type2", iicfg.indexinfo().get(1).name()); + assertEquals("type3", iicfg.indexinfo().get(2).name()); + } + { + AttributesConfig rac1 = model.getConfig(AttributesConfig.class, "test/search/cluster.test/type1"); + assertEquals(1, rac1.attribute().size()); + assertEquals("f2", rac1.attribute(0).name()); + AttributesConfig rac2 = model.getConfig(AttributesConfig.class, "test/search/cluster.test/type2"); + assertEquals(1, rac2.attribute().size()); + assertEquals("f4", rac2.attribute(0).name()); + } + { + IlscriptsConfig icfg = model.getConfig(IlscriptsConfig.class, "test/search/cluster.test"); + assertEquals(3, icfg.ilscript().size()); + assertEquals("type1", icfg.ilscript(0).doctype()); + assertEquals("type2", icfg.ilscript(1).doctype()); + assertEquals("type3", icfg.ilscript(2).doctype()); + } + } + + @Test + public void requireThatRelevantConfigIsAvailableForClusterSearcher() throws ParseException, IOException, SAXException { + final List<String> sds = Arrays.asList("type1", "type2"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "index"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; + + { // documentdb-info config + DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, searcherId); + assertEquals(2, dcfg.documentdb().size()); + + { // type1 + DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); + assertEquals("type1", db.name()); + assertEquals(6, db.rankprofile().size()); + + assertRankProfile(db, 0, "default", false, false); + assertRankProfile(db, 1, "unranked", false, false); + assertRankProfile(db, 2, "staticrank", false, false); + assertRankProfile(db, 3, "summaryfeatures", true, false); + assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); + assertRankProfile(db, 5, "rankfeatures", false, true); + + + assertEquals(2, db.summaryclass().size()); + assertEquals("default", db.summaryclass(0).name()); + assertEquals("attributeprefetch", db.summaryclass(1).name()); + assertSummaryField(db, 0, 0, "f1", "longstring", true); + assertSummaryField(db, 0, 1, "f2", "integer", false); + } + { // type2 + DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1); + assertEquals("type2", db.name()); + } + } + { // attributes config + AttributesConfig acfg = model.getConfig(AttributesConfig.class, searcherId); + assertEquals(2, acfg.attribute().size()); + assertEquals("f2", acfg.attribute(0).name()); + assertEquals("f4", acfg.attribute(1).name()); + assertEquals("f4", acfg.attribute(1).name()); + } + } + + private void assertRankProfile(DocumentdbInfoConfig.Documentdb db, int index, String name, + boolean hasSummaryFeatures, boolean hasRankFeatures) { + DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile0 = db.rankprofile(index); + assertEquals(name, rankProfile0.name()); + assertEquals(hasSummaryFeatures, rankProfile0.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile0.hasRankFeatures()); + } + + private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + + + @Test + public void requireThatConfigIsAvailableForStreaming() throws ParseException, IOException, SAXException { + final List<String> sds = Arrays.asList("type"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, null, "streaming"), ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + + DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); + assertEquals(1, dcfg.documentdb().size()); + DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); + assertEquals("type", db.name()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentSelectionConverterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentSelectionConverterTest.java new file mode 100644 index 00000000000..6941616acc8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentSelectionConverterTest.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.vespa.model.search.DocumentSelectionConverter; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for RemoveSelection. + * @author lulf + */ +public class DocumentSelectionConverterTest { + @Test + public void testQueryConversion() throws ParseException, IllegalArgumentException, UnsupportedOperationException { + DocumentSelectionConverter converter = new DocumentSelectionConverter("music.expire>now() - 3600 and video.expire > now() - 300"); + assertEquals("expire:>now(3600)", converter.getQuery("music")); + assertEquals("expire:<now(3600)", converter.getInvertedQuery("music")); + assertEquals("expire:>now(300)", converter.getQuery("video")); + assertEquals("expire:<now(300)", converter.getInvertedQuery("video")); + assertTrue(null == converter.getQuery("book")); + assertTrue(null == converter.getInvertedQuery("book")); + } + @Test + public void testSelection() throws ParseException, IllegalArgumentException, UnsupportedOperationException { + DocumentSelectionConverter converter = new DocumentSelectionConverter("music.expire>music.expire.nowdate"); + assertTrue(converter.getQuery("music") == null); + assertTrue(converter.getInvertedQuery("music") == null); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java new file mode 100644 index 00000000000..68deb96e632 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java @@ -0,0 +1,187 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.document.DataType; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * + * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and + * write _unit_ tests. Thanks. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class SearchClusterTest { + + private String vespaHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts> " + + "<host name=\"foo\">" + + "<alias>node0</alias>" + + "</host>" + + "<host name=\"bar\">" + + "<alias>node1</alias>" + + "</host>" + + "<host name=\"baz\">" + + "<alias>node2</alias>" + + "</host>" + + "</hosts>"; + + @Test + public void testSdConfigLogical() throws IOException, SAXException { + String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>" + + " <search version=\"2.0\">" + + " <qrservers>" + + " <qrserver hostalias=\"node0\" />" + + " </qrservers>" + + " <cluster name=\"s1\" indexingmode=\"realtime\">" + + " <searchdefinitions>" + + " <searchdefinition name=\"s1\" />" + + " <searchdefinition name=\"s2\" />" + + " </searchdefinitions>" + + " <documents selection=\"music\" feedname=\"a\" />" + + " <row index=\"0\">" + + " <searchnodes>" + + " <searchnode hostalias=\"node2\" index=\"0\" />" + + " </searchnodes>" + + " </row>" + + " </cluster>" + + " </search>" + + "</services>"; + ApplicationPackage app = new VespaModelCreatorWithMockPkg(vespaHosts, services).appPkg; + + // sd1 + SDDocumentType sdt1=new SDDocumentType("s1"); + Search search1 = new Search("s1", null); + SDField f1=new SDField("f1", DataType.STRING); + f1.addAttribute(new Attribute("f1", DataType.STRING)); + f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); + sdt1.addField(f1); + search1.addDocument(sdt1); + + // sd2 + SDDocumentType sdt2=new SDDocumentType("s2"); + Search search2 = new Search("s2", null); + SDField f2=new SDField("f2", DataType.STRING); + f2.addAttribute(new Attribute("f2", DataType.STRING)); + f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); + sdt2.addField(f2); + search2.addDocument(sdt2); + + SearchBuilder builder = new SearchBuilder(); + builder.importRawSearch(search1); + builder.importRawSearch(search2); + builder.build(); + } + + @Test + public void search_model_is_connected_to_container_clusters_two_content_clusters() throws Exception { + String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>\n" + + " <jdisc version='1.0' id='j1'>\n" + + " <search>" + + " <chain id='s1Chain'>" + + " <searcher id='S1ClusterSearcher'/>" + + " </chain>" + + " <provider cluster='normal' id='normal' type='local'/>\n" + + " </search>" + + " <nodes>" + + " <node hostalias=\"node0\" />" + + " </nodes>" + + " </jdisc>" + + + " <jdisc version='1.0' id='j2'>" + + " <search>" + + " <chain id='s2Chain'>" + + " <searcher id='S2ClusterSearcher'/>" + + " </chain>" + + " <provider cluster='xbulk' id='xbulk' type='local'/>" + + " </search>" + + " <nodes>" + + " <node hostalias=\"node2\" />" + + " </nodes>" + + " </jdisc>" + + + " <content id='xbulk' version=\"1.0\">" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document mode='index' type=\"music\" />" + + " </documents>" + + " <nodes>" + + " <node hostalias=\"node0\" distribution-key=\"0\" />" + + " </nodes>" + + " </content>" + + " <content id=\"normal\" version='1.0'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document mode='index' type=\"music\" />" + + " </documents>" + + " <nodes>" + + " <node hostalias=\"node2\" distribution-key=\"0\" />" + + " </nodes>" + + " </content>" + + "</services>"; + + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + + ContainerCluster cluster1 = (ContainerCluster)model.getConfigProducer("j1").get(); + assertFalse(cluster1.getSearch().getChains().localProviders().isEmpty()); + + ContainerCluster cluster2 = (ContainerCluster)model.getConfigProducer("j2").get(); + assertFalse(cluster2.getSearch().getChains().localProviders().isEmpty()); + + QrSearchersConfig.Builder builder = new QrSearchersConfig.Builder(); + cluster1.getConfig(builder); + QrSearchersConfig config = new QrSearchersConfig(builder); + System.out.println(config); + + assertThat(config.searchcluster().size(), is(2)); + int normalId = 0; + int bulkId = 1; + assertThat(config.searchcluster().get(normalId).name(), is("normal")); + assertThat(config.searchcluster().get(bulkId).name(), is("xbulk")); + + ClusterConfig.Builder clusterConfigBuilder = new ClusterConfig.Builder(); + model.getConfig(clusterConfigBuilder, "j1/searchchains/chain/normal/component/com.yahoo.prelude.cluster.ClusterSearcher"); + ClusterConfig clusterConfig = new ClusterConfig(clusterConfigBuilder); + System.out.println(clusterConfig); + assertThat(clusterConfig.clusterId(), is(normalId)); + assertThat(clusterConfig.clusterName(), is("normal")); + + ClusterConfig.Builder clusterConfigBuilder2 = new ClusterConfig.Builder(); + model.getConfig(clusterConfigBuilder2, "j2/searchchains/chain/xbulk/component/com.yahoo.prelude.cluster.ClusterSearcher"); + ClusterConfig clusterConfig2 = new ClusterConfig(clusterConfigBuilder2); + System.out.println(clusterConfig2); + assertThat(clusterConfig2.clusterId(), is(bulkId)); + assertThat(clusterConfig2.clusterName(), is("xbulk")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java new file mode 100644 index 00000000000..e9aa8f2267f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.search.NodeSpec; +import com.yahoo.vespa.model.search.SearchNode; +import com.yahoo.vespa.model.search.TransactionLogServer; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertTrue; + + +/** + * Unit tests for search node. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class SearchNodeTest { + + private void assertBaseDir(String expected, SearchNode node) { + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + node.getConfig(builder); + ProtonConfig cfg = new ProtonConfig(builder); + assertEquals(expected, cfg.basedir()); + } + + private void prepare(MockRoot root, SearchNode node) { + Host host = new Host(root, "mockhost"); + TransactionLogServer tls = new TransactionLogServer(root, "mycluster"); + tls.setHostResource(new HostResource(host)); + tls.setBasePort(100); + tls.initService(); + node.setTls(tls); + node.setHostResource(new HostResource(host)); + node.setBasePort(200); + node.initService(); + root.freezeModelTopology(); + } + + @Test + public void requireThatBasedirIsCorrectForElasticMode() { + MockRoot root = new MockRoot(""); + SearchNode node = SearchNode.create(root, "mynode", 3, new NodeSpec(7, 5), "mycluster", null, false); + prepare(root, node); + assertBaseDir(Defaults.getDefaults().vespaHome() + "var/db/vespa/search/cluster.mycluster/n3", node); + } + + @Test + public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { + MockRoot root = new MockRoot(""); + SearchNode node = SearchNode.create(root, "mynode", 3, new NodeSpec(7, 5), "mycluster", null, false); + node.setHostResource(new HostResource(new Host(node, "mynbode"))); + node.initService(); + assertFalse(node.getPreShutdownCommand().isPresent()); + } + + @Test + public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { + MockRoot root = new MockRoot(""); + SearchNode node = SearchNode.create(root, "mynode2", 4, new NodeSpec(7, 5), "mycluster", null, true); + node.setHostResource(new HostResource(new Host(node, "mynbode2"))); + node.initService(); + assertTrue(node.getPreShutdownCommand().isPresent()); + Assert.assertThat(node.getPreShutdownCommand().get(), + CoreMatchers.containsString("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java new file mode 100644 index 00000000000..558fd4f9a80 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.utils; + +import com.yahoo.vespa.config.search.core.FdispatchrcConfig; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.model.search.Dispatch; + +import static org.junit.Assert.assertEquals; + +public class DispatchUtils { + + public static PartitionsConfig.Dataset getDataset(Dispatch dispatch) { + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + dispatch.getConfig(builder); + PartitionsConfig cfg = new PartitionsConfig(builder); + assertEquals(1, cfg.dataset().size()); + return cfg.dataset(0); + } + + public static FdispatchrcConfig getFdispatchrcConfig(Dispatch dispatch) { + FdispatchrcConfig.Builder builder = new FdispatchrcConfig.Builder(); + dispatch.getConfig(builder); + return new FdispatchrcConfig(builder); + } + + public static void assertEngine(int rowId, int partitionId, PartitionsConfig.Dataset.Engine engine) { + assertEquals(rowId, engine.rowid()); + assertEquals(partitionId, engine.partid()); + } + + public static void assertEngine(int rowId, int partitionId, String connectSpec, PartitionsConfig.Dataset.Engine engine) { + assertEngine(rowId, partitionId, engine); + assertEquals(connectSpec, engine.name_and_port()); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/storage/DistributionBitCalculatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/storage/DistributionBitCalculatorTest.java new file mode 100644 index 00000000000..6d6c31ac0d2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/storage/DistributionBitCalculatorTest.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.storage; + +import com.yahoo.vespa.model.content.DistributionBitCalculator; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class DistributionBitCalculatorTest { + + @Test + public void testBitCalculator() { + ContentCluster.DistributionMode mode = ContentCluster.DistributionMode.STRICT; + assertEquals(8, DistributionBitCalculator.getDistributionBits(1, mode)); + assertEquals(16, DistributionBitCalculator.getDistributionBits(10, mode)); + assertEquals(21, DistributionBitCalculator.getDistributionBits(100, mode)); + assertEquals(25, DistributionBitCalculator.getDistributionBits(500, mode)); + assertEquals(28, DistributionBitCalculator.getDistributionBits(1000, mode)); + + mode = ContentCluster.DistributionMode.LOOSE; + assertEquals( 8, DistributionBitCalculator.getDistributionBits(1, mode)); + assertEquals( 8, DistributionBitCalculator.getDistributionBits(4, mode)); + assertEquals(16, DistributionBitCalculator.getDistributionBits(5, mode)); + assertEquals(16, DistributionBitCalculator.getDistributionBits(199, mode)); + assertEquals(24, DistributionBitCalculator.getDistributionBits(200, mode)); + assertEquals(24, DistributionBitCalculator.getDistributionBits(2500, mode)); + + mode = ContentCluster.DistributionMode.LEGACY; + assertEquals( 1, DistributionBitCalculator.getDistributionBits(1, mode)); + assertEquals(14, DistributionBitCalculator.getDistributionBits(4, mode)); + assertEquals(19, DistributionBitCalculator.getDistributionBits(16, mode)); + assertEquals(23, DistributionBitCalculator.getDistributionBits(200, mode)); + assertEquals(28, DistributionBitCalculator.getDistributionBits(2500, mode)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java new file mode 100644 index 00000000000..d7e9286b854 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/storage/test/StorageModelTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.storage.test; + +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * Tests storage model + * + * + * @author gjoranv + */ +public class StorageModelTestCase { + + @Test(expected=RuntimeException.class) + public void testTwoClustersSameName() throws Exception { + createModel("src/test/cfg/storage/twoclusterssamename"); + } + + private VespaModel createModel(String filename) { + return new VespaModelCreatorWithFilePkg(filename).create(); + } + + @Test + public void testIndexGreaterThanNumNodes() throws Exception { + VespaModel vespaModel = createModel("src/test/cfg/storage/app_index_higher_than_num_nodes"); + + // Test fleet controller config + FleetcontrollerConfig fleetController1Config = new FleetcontrollerConfig((FleetcontrollerConfig.Builder) + vespaModel.getConfig(new FleetcontrollerConfig.Builder(), "content/fleetcontroller")); + + assertEquals(60000, fleetController1Config.storage_transition_time()); + assertEquals(8, fleetController1Config.ideal_distribution_bits()); + } + + @Test + public void testMetricsSnapshotIntervalYAMAS() throws Exception { + VespaModel vespaModel = createModel("src/test/cfg/storage/clustercontroller_advanced"); + ContentCluster contentCluster = vespaModel.getContentClusters().values().iterator().next(); + assertNotNull(contentCluster); + MetricsmanagerConfig.Builder builder = new MetricsmanagerConfig.Builder(); + contentCluster.getConfig(builder); + MetricsmanagerConfig config = new MetricsmanagerConfig(builder); + assertThat(config.snapshot().periods(0), is(60)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java new file mode 100644 index 00000000000..af9dc275922 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.builder.xml.dom.LegacyConfigModelBuilder; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * This is a plugin for testing the plugin API exchange mechanism in + * the vespamodel. It uses the API of another plugin. + * + * @author gjoranv + */ +public class ApiConfigModel extends ConfigModel { + + private List<ApiService> apiServices = new ArrayList<>(); + + public ApiConfigModel(ConfigModelContext modelContext) { + super(modelContext); + } + + // Inherit doc from ConfigModel. + public void prepare(ConfigModelRepo configModelRepo) { + int numSimpleServices = 0; + ConfigModel simplePlugin = configModelRepo.get("simple"); + + if ((simplePlugin != null) && (simplePlugin instanceof TestApi)) { + TestApi testApi = (TestApi) simplePlugin; + numSimpleServices = testApi.getNumSimpleServices(); + } + for (Object apiService : apiServices) { + ApiService as = (ApiService) apiService; + as.setNumSimpleServices(numSimpleServices); + } + } + + public static class Builder extends LegacyConfigModelBuilder<ApiConfigModel> { + + public Builder() { + super(ApiConfigModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromName("api")); + } + + @Override + public void doBuild(ApiConfigModel configModel, Element spec, ConfigModelContext modelContext) { + NodeList pl = spec.getElementsByTagName("apiservice"); + if (pl.getLength() > 0) { + for (int i=0; i < pl.getLength(); i++) { + configModel.apiServices.add(new DomTestServiceBuilder.ApiServiceBuilder(i).build(modelContext.getParentProducer(), + (Element) pl.item(i))); + } + } + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java new file mode 100644 index 00000000000..e0756fec650 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiService.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +/** + * This is a service for testing the plugin exchange mechanism in the + * vespamodel. It provides some data that are made public in the API + * of the plugin that owns it. + * + * @author gjoranv + */ +public class ApiService extends AbstractService implements com.yahoo.test.StandardConfig.Producer { + + private int numSimpleServices = 0; + + /** + * Creates a new ApiService instance + * + * @param parent The parent ConfigProducer. + * @param name Service name + */ + public ApiService(AbstractConfigProducer<?> parent, String name) { + super(parent, name); + } + + public void getConfig(com.yahoo.test.StandardConfig.Builder builder) { + builder.astring("apiservice"); + + } + + public void setNumSimpleServices(int nss) { + numSimpleServices = nss; + } + + public int getPortCount() { return 0; } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/DomTestServiceBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/test/DomTestServiceBuilder.java new file mode 100644 index 00000000000..02a1318fa56 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/DomTestServiceBuilder.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import org.w3c.dom.Element; + +/** + * Builders for test services + */ +public class DomTestServiceBuilder { + + + static class SimpleServiceBuilder + extends VespaDomBuilder.DomConfigProducerBuilder<SimpleService> { + int i; + + public SimpleServiceBuilder(int i) { + this.i = i; + } + + @Override + protected SimpleService doBuild(AbstractConfigProducer parent, + Element spec) { + return new SimpleService(parent, "simpleservice." + i); + } + } + + static class ApiServiceBuilder + extends VespaDomBuilder.DomConfigProducerBuilder<ApiService> { + int i; + + public ApiServiceBuilder(int i) { + this.i = i; + } + + @Override + protected ApiService doBuild(AbstractConfigProducer parent, + Element spec) { + return new ApiService(parent, "apiservice." + i); + } + } + + static class ParentServiceBuilder + extends VespaDomBuilder.DomConfigProducerBuilder<ParentService> { + int i; + + public ParentServiceBuilder(int i) { + this.i = i; + } + + @Override + protected ParentService doBuild(AbstractConfigProducer parent, + Element spec) { + return new ParentService(parent, "parentservice." + i, spec); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java new file mode 100644 index 00000000000..1032f5099c6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.builder.xml.dom.DomContentBuilder; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.content.Content; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Demonstrates how a model can be added at build time to amend another model. + * This is useful is situations where the core Vespa config models needs to be + * modified by third party code which follows the installed environment rather than + * the application. + * + * @author bratseth + */ +public class ModelAmendingTestCase { + + @Test + public void testModelAmending() throws IOException, SAXException { + ConfigModelRegistry amendingModelRepo = MapConfigModelRegistry.createFromList(new ContainerModelAmenderBuilder(), + new ContentModelAmenderBuilder()); + VespaModel model = new VespaModel(new MockApplicationPackage.Builder() + .withServices( + "<services version='1.0'>" + + " <jdisc id='test1' version='1.0'>" + + " <search />" + + " </jdisc>" + + " <jdisc id='test2' version='1.0'>" + + " <http><server id='server1' port='19107'/></http>" + + " <document-api/>" + + " </jdisc>" + + " <content id='test3' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document mode='index' type='testtype1'/>" + + " </documents>" + + " </content>" + + " <content id='test4' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document mode='index' type='testtype1'/>" + + " </documents>" + + " </content>" + + "</services>") + .withSearchDefinitions( + searchDefinition("testtype1")) + .build(), + amendingModelRepo); + assertEquals(1, model.getHostSystem().getHosts().size()); + + // Check that explicit jdisc clusters are amended + assertEquals(4, model.getContainerClusters().size()); + assertNotNull(model.getContainerClusters().get("test1").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); + assertNotNull(model.getContainerClusters().get("test2").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); + assertNotNull(model.getContainerClusters().get("cluster.test3.indexing").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); + assertNotNull(model.getContainerClusters().get("cluster.test4.indexing").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); + } + + private List<String> searchDefinition(String name) { + return Collections.singletonList( + "search " + name + " {" + + " document " + name + " {" + + " field testfield type string {}" + + " }" + + "}"); + } + + public static class ContainerModelAmenderBuilder extends ConfigModelBuilder<ContainerModelAmender> { + + private boolean built = false; + + public ContainerModelAmenderBuilder() { + super(ContainerModelAmender.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return ContainerModelBuilder.configModelIds; + } + + @Override + public void doBuild(ContainerModelAmender model, Element spec, ConfigModelContext modelContext) { + if (built) return; // the same instance will be called once per jdisc cluster + for (ContainerModel containerModel : model.containerModels) + amend(containerModel.getCluster()); + built = true; + } + + static void amend(ContainerCluster cluster) { + cluster.addSimpleComponent("com.yahoo.MyAmendedComponent", null, "my-amendment-bundle"); + } + + } + + public static class ContainerModelAmender extends ConfigModel { + + /** The container models this builder amends */ + private final Collection<ContainerModel> containerModels; + + public ContainerModelAmender(ConfigModelContext modelContext, Collection<ContainerModel> containerModels) { + super(modelContext); + this.containerModels = containerModels; + } + + @Override + public boolean isServing() { return false; } + + } + + public static class ContentModelAmenderBuilder extends ConfigModelBuilder<ContentModelAmender> { + + private boolean built = false; + + public ContentModelAmenderBuilder() { + super(ContentModelAmender.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return DomContentBuilder.configModelIds; + } + + @Override + public void doBuild(ContentModelAmender model, Element spec, ConfigModelContext modelContext) { + if (built) return; // the same instance will be called once per content cluster + for (Content contentModel : model.contentModels) + contentModel.ownedIndexingCluster().ifPresent(ContainerModelAmenderBuilder::amend); + built = true; + } + } + + public static class ContentModelAmender extends ConfigModel { + + private final Collection<Content> contentModels; + + public ContentModelAmender(ConfigModelContext modelContext, Collection<Content> contentModels) { + super(modelContext); + this.contentModels = contentModels; + } + + @Override + public boolean isServing() { return false; } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelConfigProviderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelConfigProviderTest.java new file mode 100644 index 00000000000..1e5a5255321 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelConfigProviderTest.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Test HostSystem + * + * @author musum + */ +public class ModelConfigProviderTest { + + /** + * Get the config via ConfigInstance based API, by getting whole config + */ + @Test + public void testGetModelConfig() { + VespaModel vespaModel = new VespaModelCreatorWithFilePkg("src/test/cfg/admin/adminconfig20").create(); + ModelConfig config = vespaModel.getConfig(ModelConfig.class, ""); + assertEquals(config.hosts().size(), 1); + ModelConfig.Hosts localhost = config.hosts(0); //Actually set to hostname. + int numLogservers=0; + int numSlobroks=0; + for (ModelConfig.Hosts.Services service : localhost.services()) { + if ("logserver".equals(service.type())) { + numLogservers++; + } + if ("slobrok".equals(service.type())) { + numSlobroks++; + } + } + assertEquals(1, numLogservers); + assertEquals(2, numSlobroks); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java new file mode 100644 index 00000000000..f1ec51a1200 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ParentService.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.test.StandardConfig.Builder; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * This is a service that creates child services + */ +public class ParentService extends AbstractService implements com.yahoo.test.StandardConfig.Producer { + + public int childCnt = 0; + + /** + * Creates a new ParentService instance + * + * @param parent The parent ConfigProducer. + * @param name Service name + * @param config The xml config Element for this Service + */ + public ParentService(AbstractConfigProducer parent, String name, + Element config) + { + super(parent, name); + + int s,p; s=p=0; + NodeList childNodes = config.getChildNodes(); + for (int i=0; i < childNodes.getLength(); i++) { + Node child = childNodes.item(i); + if (! (child instanceof Element)) { + // skip #text and #comment nodes + continue; + } + Element e = (Element)child; + String service = e.getTagName(); + + if (service.equals("simpleservice")) { + new SimpleService(this, "simpleservice."+s); + s++; + } + else if (service.equals("parentservice")) { + new ParentService(this, "parentservice."+p, e); + p++; + } + else { + throw new IllegalArgumentException("Unknown service: " + service); + } + } + } + + @Override + public void getConfig(Builder builder) { + builder.astring("parentservice"); + } + + public int getPortCount() { return 0; } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java new file mode 100644 index 00000000000..40dc134190b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.vespa.model.PortsMeta; + +/** + * Tests proper functioning of the PortsMeta. + * + * + * @author Vidar Larsen + */ +public class PortsMetaTestCase extends junit.framework.TestCase { + public void testRegister() throws Exception { + PortsMeta pm = new PortsMeta(); + pm.on(0).tag("foo"); + pm.on(1).tag("bar"); + pm.on(5).tag("xyzzy"); + + assertTrue(pm.contains(0, "foo")); + assertTrue(pm.contains(1, "bar")); + assertTrue(pm.contains(5, "xyzzy")); + assertFalse(pm.contains(0, "bar")); + assertFalse(pm.contains(2, "anything")); + } + public void testAdminStatusApi() throws Exception { + PortsMeta pm = new PortsMeta() + .on(0).tag("rpc").tag("nc").tag("admin").tag("status") + .on(1).tag("rpc").tag("rtx").tag("admin").tag("status") + .on(2).tag("http").tag("admin"); + + assertEquals(1, pm.getRpcAdminOffset().intValue()); + assertEquals(1, pm.getRpcStatusOffset().intValue()); + assertEquals(2, pm.getHttpAdminOffset().intValue()); + assertNull(pm.getHttpStatusOffset()); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java new file mode 100644 index 00000000000..ca1cc6a3500 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.builder.xml.dom.LegacyConfigModelBuilder; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * A simple test config model. + * + * @author gjoranv + */ +public class SimpleConfigModel extends ConfigModel implements TestApi { + + private List<SimpleService> simpleServices = new ArrayList<>(); + private List<ParentService> parentServices = new ArrayList<>(); + + public SimpleConfigModel(ConfigModelContext modelContext) { + super(modelContext); + } + + /** Implement TestApi */ + public int getNumSimpleServices() { + return simpleServices.size(); + } + public int getNumParentServices() { + return parentServices.size(); + } + + public static class Builder extends LegacyConfigModelBuilder<SimpleConfigModel> { + + public Builder() { + super(SimpleConfigModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromName("simple")); + } + + @Override + public void doBuild(SimpleConfigModel configModel, Element spec, ConfigModelContext modelContext) { + int s,p; s=p=0; + + // Validate the services given in the config + NodeList childNodes = spec.getChildNodes(); + for (int i=0; i < childNodes.getLength(); i++) { + Node child = childNodes.item(i); + if (! (child instanceof Element)) { + // skip #text and #comment nodes + continue; + } + Element e = (Element)child; + String service = e.getTagName(); + + if (service.equals("simpleservice")) { + configModel.simpleServices.add(new DomTestServiceBuilder.SimpleServiceBuilder(s).build(modelContext.getParentProducer(), e)); + s++; + } + else if (service.equals("parentservice")) { + configModel.parentServices.add(new DomTestServiceBuilder.ParentServiceBuilder(p).build(modelContext.getParentProducer(), e)); + p++; + } + else { + throw new IllegalArgumentException("Unknown service: " + service); + } + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java new file mode 100644 index 00000000000..cfef392f2c4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleService.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.test.StandardConfig.Builder; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +import java.util.HashMap; + +/** + * This service has a desired default port and returns the actual + * baseport from getConfig(). + * + * @author gjoranv + */ +public class SimpleService extends AbstractService implements com.yahoo.test.StandardConfig.Producer { + + /** + * Creates a new SimpleService instance + * + * @param parent The parent ConfigProducer. + * @param name Service name + */ + public SimpleService(AbstractConfigProducer parent, String name) { + super(parent, name); + portsMeta.on(0).tag("base") + .on(1).tag("base") + .on(2).tag("base") + .on(3).tag("base") + .on(4).tag("base"); + } + + @Override + public void getConfig(Builder builder) { + builder.astring("simpleservice").baseport(getRelativePort(0)); + } + + public int getWantedPort(){ return 10000; } + public int getPortCount() { return 5; } + + // Make sure this service is listed in the sentinel config + public String getStartupCommand() { return "sleep 0"; } + + public boolean getAutostartFlag() { return false; } + public boolean getAutorestartFlag() { return false; } + + @Override + public HashMap<String,String> getDefaultMetricDimensions(){ + HashMap<String, String> dimensions = new HashMap<>(); + dimensions.put("clustername", "testClusterName"); + return dimensions; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/TestApi.java b/config-model/src/test/java/com/yahoo/vespa/model/test/TestApi.java new file mode 100644 index 00000000000..b0218769e25 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/TestApi.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +/** + * This is a simple API for testing the plugin api exchange mechanism. + * + * @author gjoranv + */ +public interface TestApi { + public int getNumSimpleServices(); + public int getNumParentServices(); +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java new file mode 100644 index 00000000000..a86dc68d9dc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -0,0 +1,341 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.net.HostName; +import com.yahoo.net.LinuxInetAddress; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.config.UnknownConfigIdException; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.Configserver; +import com.yahoo.vespa.model.application.validation.Validation; +import com.yahoo.vespa.model.test.utils.CommonVespaModelSetup; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Ignore; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public class VespaModelTestCase { + + private static final String TESTDIR = "src/test/cfg/application/"; + private static final String simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts> " + + "<host name=\"localhost\">" + + "<alias>node0</alias>" + + "</host>" + + "</hosts>"; + + public static VespaModel getVespaModel(String configPath) { + return getVespaModel(configPath, true); + } + + public static VespaModel getVespaModel(String configPath, boolean validateXml) { + VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg(configPath); + return creator.create(validateXml); + } + + // Debugging + @SuppressWarnings({"UnusedDeclaration"}) + private static void dumpTree(ConfigProducer producer) { + Map<String, ? extends ConfigProducer> id2cp = producer.getChildren(); + for (ConfigProducer c : id2cp.values()) { + System.out.println("id: " + c.getConfigId()); + if (c.getChildren().size() > 0) { + dumpTree(c); + } + } + } + + // Verify that common config from plugins is delivered from the root node for any configId, using the Builder based API + @Test + public void testCommonConfig() throws Exception { + VespaModel model = getVespaModel(TESTDIR + "app_nohosts/"); + LogdConfig.Builder b = new LogdConfig.Builder(); + b = (LogdConfig.Builder) model.getConfig(b, ""); + LogdConfig c = new LogdConfig(b); + assertEquals(c.logserver().host(), LinuxInetAddress.getLocalHost().getCanonicalHostName()); + + SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); + sb = (com.yahoo.cloud.config.SlobroksConfig.Builder) model.getConfig(sb, ""); + SlobroksConfig sbc = new SlobroksConfig(sb); + assertEquals(sbc.slobrok().size(), 1); + + ZookeepersConfig.Builder zb = new ZookeepersConfig.Builder(); + zb = (ZookeepersConfig.Builder) model.getConfig(zb, ""); + ZookeepersConfig zc = new ZookeepersConfig(zb); + assertEquals(zc.zookeeperserverlist().split(",").length, 2); + assertTrue(zc.zookeeperserverlist().startsWith(LinuxInetAddress.getLocalHost().getCanonicalHostName())); + + ApplicationIdConfig.Builder appIdBuilder = new ApplicationIdConfig.Builder(); + appIdBuilder = (ApplicationIdConfig.Builder) model.getConfig(appIdBuilder, ""); + ApplicationIdConfig applicationIdConfig = new ApplicationIdConfig(appIdBuilder); + assertEquals(ApplicationId.defaultId().tenant().value(), applicationIdConfig.tenant()); + assertEquals(ApplicationId.defaultId().application().value(), applicationIdConfig.application()); + assertEquals(ApplicationId.defaultId().instance().value(), applicationIdConfig.instance()); + } + + @Test + public void testHostsConfig() { + VespaModel model = getVespaModel(TESTDIR + "app_qrserverandgw"); + LogdConfig config = getLogdConfig(model, ""); + assertEquals(config.logserver().host(), HostName.getLocalhost()); + assertNotNull(config); + config = getLogdConfig(model, "hosts"); + assertNotNull(config); + assertEquals(config.logserver().host(), HostName.getLocalhost()); + } + + private static LogdConfig getLogdConfig(VespaModel model, String configId) { + LogdConfig.Builder b = new LogdConfig.Builder(); + b = (LogdConfig.Builder) model.getConfig(b, configId); + if (b == null) + return null; + return new LogdConfig(b); + } + + @Test + public void testHostsOverrides() throws IOException, SAXException { + VespaModel model = new VespaModelCreatorWithMockPkg( + simpleHosts, + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + "<config name=\"cloud.config.log.logd\">" + + "<logserver><host>foo</host></logserver>" + + "</config>" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\" />" + + "</admin>" + + "</services>").create(); + LogdConfig config = getLogdConfig(model, ""); + assertNotNull(config); + assertEquals(config.logserver().host(), "foo"); + config = getLogdConfig(model, "hosts/" + HostName.getLocalhost() + "/logd"); + assertNotNull(config); + assertEquals(config.logserver().host(), "foo"); + } + + @Ignore + @Test(expected = UnknownConfigIdException.class) + public void testIllegalConfigIdWithBuilders() { + VespaModel model = getVespaModel(TESTDIR + "app_nohosts/"); + DocumentmanagerConfig.Builder db = new DocumentmanagerConfig.Builder(); + model.getConfig(db, "bogus"); + } + + @Test + public void testConfigLists() { + VespaModel model = getVespaModel(TESTDIR + "app_nohosts/"); + assertTrue(model.allConfigsProduced().size() > 0); + assertTrue(model.allConfigIds().size() > 0); + } + + @Test + public void testCreateFromReaders() throws SAXException, IOException { + VespaModel model = CommonVespaModelSetup.createVespaModelWithMusic( + simpleHosts, + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\" />" + + "</admin>" + + "<container version=\"1.0\">" + + " <nodes>" + + " <node hostalias=\"node0\" />" + + " </nodes>" + + " <search/>" + + " <document-api/>" + + "</container>" + + "<content id=\"music\" version=\"1.0\">" + + " <redundancy>1</redundancy>" + + " <nodes>" + + " <node hostalias=\"node0\" distribution-key=\"0\"/>" + + " </nodes>" + + " <documents>" + + " <document type=\"music\" mode=\"index\"/>" + + " </documents>" + + "</content>" + + "</services>"); + ContainerHttpConfig container = new ContainerHttpConfig((ContainerHttpConfig.Builder) model.getConfig(new ContainerHttpConfig.Builder(), "container/container.0")); + assertEquals(container.port().search(), Defaults.getDefaults().vespaWebServicePort()); + MessagebusConfig.Builder mBusB = new MessagebusConfig.Builder(); + model.getConfig(mBusB, "client"); + MessagebusConfig mBus = new MessagebusConfig(mBusB); + assertEquals(mBus.routingtable().size(), 1); + } + + @Test(expected = IllegalArgumentException.class) + public void testHostsWithoutAliases() { + new TestDriver().buildModel( + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>" + + "</services>", + "<hosts>" + + " <host name='localhost'>" + + " <alias>node0</alias>" + + " </host>" + + " <host name='foo.yahoo.com' />" + + "</hosts>"); + } + + class MyLogger implements DeployLogger { + List<Pair<Level, String>> msgs = new ArrayList<>(); + @Override + public void log(Level level, String message) { + msgs.add(new Pair<>(level, message)); + } + } + + @Test + public void testDeployLogger() throws IOException, SAXException { + final String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + "<config name=\"unknsownfoo\">" + + "<logserver><host>foo</host></logserver>" + + "</config>" + + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\" />" + + "</admin>" + + "</services>"; + + MyLogger logger = new MyLogger(); + final DeployState.Builder builder = new DeployState.Builder(); + builder.modelHostProvisioner(new HostsXmlProvisioner(new StringReader(simpleHosts))); + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts(simpleHosts) + .withServices(services) + .build(); + DeployState deployState = builder.deployLogger(logger).applicationPackage(app).build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + Validation.validate(model, true, deployState); + System.out.println(logger.msgs); + assertFalse(logger.msgs.isEmpty()); + } + + @Test + public void testNoAdmin() throws IOException, SAXException { + VespaModel model = CommonVespaModelSetup.createVespaModelWithMusic( + simpleHosts, + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<services version=\"1.0\">" + + "</services>"); + Admin admin = model.getAdmin(); + assertThat(admin.getSlobroks().size(), is(1)); + assertThat(admin.getConfigservers().size(), is(1)); + Set<HostInfo> hosts = model.getHosts(); + assertThat(hosts.size(), is(1)); + //logd, config proxy, sentinel, config server, slobrok, log server, file distributor + HostInfo host = hosts.iterator().next(); + assertThat(host.getServices().size(), is(7)); + new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); + + } + + @Test + public void testNoMultitenantHostExported() throws IOException, SAXException { + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withServices("<services version='1.0'><admin version='3.0'><nodes count='1' /></admin></services>") + .build(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com")) + .properties(new DeployProperties.Builder() + .configServerSpecs(Arrays.asList(new Configserver.Spec("cfghost", 1234, 1235, 1236))) + .multitenant(true) + .build()) + .build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + ProvisionInfo info = model.getProvisionInfo().get(); + assertEquals("Admin version 3 is ignored, and there are no other hosts to borrow for admin services", 0, info.getHosts().size()); + } + + @Test + public void testMinimalApp() throws IOException, SAXException { + VespaModel model = new VespaModel(new MockApplicationPackage.Builder() + .withServices("<services version='1.0'><jdisc version='1.0'><search /></jdisc></services>") + .build()); + assertThat(model.getHostSystem().getHosts().size(), is(1)); + assertThat(model.getContainerClusters().size(), is(1)); + } + + @Test + public void testPermanentServices() throws IOException, SAXException { + ApplicationPackage app = MockApplicationPackage.createEmpty(); + DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), builder.build()); + assertThat(model.getContainerClusters().size(), is(0)); + model = new VespaModel(new NullConfigModelRegistry(), builder.permanentApplicationPackage(Optional.of(FilesApplicationPackage.fromFile(new File(TESTDIR, "app_permanent")))).build()); + assertThat(model.getContainerClusters().size(), is(1)); + } + + @Test + public void testConfigResolving() throws IOException { + VespaModel model = VespaModelTestCase.getVespaModel(TESTDIR + "app_nohosts/"); + ConfigDefinition def = new ConfigDefinition(LogdConfig.CONFIG_DEF_NAME, LogdConfig.CONFIG_DEF_SCHEMA); + ConfigKey<?> key = new ConfigKey<>(LogdConfig.CONFIG_DEF_NAME, "", LogdConfig.CONFIG_DEF_NAMESPACE); + ConfigPayload payload = model.getConfig(key, def, null); + assertPort(payload, 19081); + + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + builder.getObject("logserver").setField("port", "19082"); + payload = model.getConfig(key, def, ConfigPayload.fromBuilder(builder)); + assertPort(payload, 19082); + payload = model.getConfig(key, def, ConfigPayload.fromBuilder(builder)); + assertPort(payload, 19082); + } + + private void assertPort(ConfigPayload payload, long expectedPort) { + long port = payload.getSlime().get().field("logserver").field("port").asLong(); + assertThat(port, is(expectedPort)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java new file mode 100644 index 00000000000..387ab07a685 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test.utils; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * For testing purposes only. + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ApplicationPackageUtils { + + public static String generateSearchDefinition(String name, String field1, String field2) { + String sd = "" + + "search " + name + "{" + + " document " + name + "{" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " header\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " header\n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + "}"; + return sd; + } + + public static Search createSearch(String name, String field1, String field2) throws ParseException { + SearchBuilder sb = new SearchBuilder(); + sb.importString(generateSearchDefinition(name, field1, field2)); + sb.build(); + return sb.getSearch(); + } + + public static SearchDefinition createSearchDefinition(String name, String field1, String field2) throws ParseException { + com.yahoo.searchdefinition.Search type = ApplicationPackageUtils.createSearch(name, field1, field2); + return new SearchDefinition(type.getName(), type); + } + + public static List<String> generateSearchDefinition(String name) { + return generateSearchDefinitions(name); + } + + public static List<String> generateSearchDefinitions(String ... sdNames) { + return generateSearchDefinitions(Arrays.asList(sdNames)); + } + + public static List<SearchDefinition> createSearchDefinition(String name) throws ParseException { + return createSearchDefinitions(Arrays.asList(name)); + } + + public static List<SearchDefinition> createSearchDefinitions(List<String> sdNames) throws ParseException { + List<SearchDefinition> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(createSearchDefinition(sdName, "f" + (i + 1), "f" + (i + 2))); + i = i + 2; + } + return sds; + } + + public static List<String> generateSearchDefinitions(List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSearchDefinition(sdName, "f" + (i + 1), "f" + (i + 2))); + i = i + 2; + } + return sds; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/CommonVespaModelSetup.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/CommonVespaModelSetup.java new file mode 100644 index 00000000000..b8388e15e0c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/CommonVespaModelSetup.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test.utils; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; + +import java.io.File; + +/** + * @author tonytv + */ +//TODO Remove, use VespaModelCreatorWithMockPkg or VespaModelCreatorWithFilePkg instead +public class CommonVespaModelSetup { + + public static VespaModel createVespaModelWithMusic(String path) { + return createVespaModelWithMusic(new File(path)); + } + + public static VespaModel createVespaModelWithMusic(File dir) { + VespaModelCreatorWithFilePkg modelCreator = new VespaModelCreatorWithFilePkg(dir); + return modelCreator.create(); + } + + public static VespaModel createVespaModelWithMusic(String hosts, String services) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withHosts(hosts) + .withServices(services) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build(); + VespaModelCreatorWithMockPkg modelCreator = new VespaModelCreatorWithMockPkg(app); + return modelCreator.create(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/DeployLoggerStub.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/DeployLoggerStub.java new file mode 100644 index 00000000000..31999a9ae51 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/DeployLoggerStub.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test.utils; + + +import com.yahoo.config.application.api.DeployLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; + +/** + * A logger stub that stores the log output to a list. + * + * @author bjorncs + */ +public class DeployLoggerStub implements DeployLogger { + + public final List<LogEntry> entries = new ArrayList<>(); + + @Override + public void log(Level level, String message) { + entries.add(new LogEntry(level, message)); + } + + public static class LogEntry { + public final Level level; + public final String message; + + public LogEntry(Level level, String message) { + this.level = level; + this.message = message; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof LogEntry)) return false; + + LogEntry logEntry = (LogEntry) o; + + if (!level.equals(logEntry.level)) return false; + if (!message.equals(logEntry.message)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = level.hashCode(); + result = 31 * result + message.hashCode(); + return result; + } + + @Override + public String toString() { + return "level='" + level + ", message='" + message + "'"; + } + } + + public LogEntry getLast() { + return entries.get(entries.size() - 1); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java new file mode 100644 index 00000000000..14216733d2a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test.utils; + +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation; + +import java.io.File; +import java.io.IOException; + +/** + * For testing purposes only + * + * @author tonytv + */ +public class VespaModelCreatorWithFilePkg { + + private FilesApplicationPackage applicationPkg; + + private ConfigModelRegistry configModelRegistry; + + public VespaModelCreatorWithFilePkg(String directoryName) { + this(new File(directoryName)); + } + + public VespaModelCreatorWithFilePkg(File directory) { + this(directory, new NullConfigModelRegistry()); + } + + public VespaModelCreatorWithFilePkg(String directoryName, ConfigModelRegistry configModelRegistry) { + this(new File(directoryName), configModelRegistry); + } + + public VespaModelCreatorWithFilePkg(File directory, ConfigModelRegistry configModelRegistry) { + this.configModelRegistry = configModelRegistry; + this.applicationPkg = FilesApplicationPackage.fromFile(directory); + } + + public VespaModel create() { + return create(true); + } + + public void validate() throws IOException { + ApplicationPackageXmlFilesValidator.createTestXmlValidator(applicationPkg.getAppDir()).checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(applicationPkg); + } + + public VespaModel create(boolean validateApplicationWithSchema) { + try { + if (validateApplicationWithSchema) { + validate(); + } + DeployState deployState = new DeployState.Builder().applicationPackage(applicationPkg).build(); + VespaModel model = new VespaModel(configModelRegistry, deployState); + // Validate, but without checking configSources or routing (routing + // is constructed in a special way and cannot always be validated in + // this step for unit tests) + Validation.validate(model, false, false, deployState); + return model; + } catch (Exception e) { + throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java new file mode 100644 index 00000000000..42d9f874c0b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.test.utils; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.application.provider.SchemaValidator; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation; + +import java.util.List; + +/** + * For testing purposes only. + * + * @author tonytv + */ +public class VespaModelCreatorWithMockPkg { + + public final ApplicationPackage appPkg; + public DeployState deployState = null; + public List<ConfigChangeAction> configChangeActions; + + public VespaModelCreatorWithMockPkg(String hosts, String services) { + this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build()); + } + + public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> searchDefinitions) { + this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSearchDefinitions(searchDefinitions).build()); + } + + public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) { + this.appPkg = appPkg; + } + + public VespaModel create() { + DeployState deployState = new DeployState.Builder().applicationPackage(appPkg).build(); + return create(true, deployState); + } + + public VespaModel create(DeployState.Builder deployStateBuilder) { + return create(true, deployStateBuilder.applicationPackage(appPkg).build()); + } + + public VespaModel create(boolean validate, DeployState deployState) { + try { + this.deployState = deployState; + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + if (validate) { + try { + SchemaValidator validator = SchemaValidator.createTestValidatorHosts(); + if (appPkg.getHosts() != null) { + validator.validate(appPkg.getHosts()); + } + validator = SchemaValidator.createTestValidatorServices(); + validator.validate(appPkg.getServices()); + } catch (Exception e) { + System.err.println(e.getClass()); + throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); + } + // Validate, but without checking configSources or routing (routing + // is constructed in a special way and cannot always be validated in + // this step for unit tests) + configChangeActions = Validation.validate(model, false, false, deployState); + } + return model; + } catch (Exception e) { + e.printStackTrace(); + throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/DurationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/DurationTest.java new file mode 100644 index 00000000000..5d056142f63 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/DurationTest.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class DurationTest { + @Test + public void testDurationUnits() { + assertEquals(1000, new Duration("1").getMilliSeconds()); + assertEquals(2.0, new Duration("2").getSeconds(), 0.0001); + assertEquals(1, new Duration("1ms").getMilliSeconds()); + assertEquals(2000, new Duration("2s").getMilliSeconds()); + assertEquals(5 * 60 * 1000, new Duration("5m").getMilliSeconds()); + assertEquals(3 * 60 * 60 * 1000, new Duration("3h").getMilliSeconds()); + assertEquals(24 * 60 * 60 * 1000, new Duration("1d").getMilliSeconds()); + + assertEquals(1400, new Duration("1.4s").getMilliSeconds()); + assertEquals(1400, new Duration("1.4 s").getMilliSeconds()); + } + + private void assertException(String str) { + try { + new Duration(str); + fail("Exception not thrown for string: " + str); + } catch (Exception e) { + } + } + + @Test + public void testParseError() { + assertException("bjarne"); + assertException(""); + assertException("1 foo"); + assertException("1.5 bar"); + assertException("-5"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java new file mode 100644 index 00000000000..41fc2edc3a7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils; + +import com.yahoo.config.FileNode; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.SimpleConfigProducer; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class FileSenderTest { + + private SimpleConfigProducer<?> producer; + private ConfigPayloadBuilder builder; + private List<AbstractService> serviceList; + private ConfigDefinition def; + private TestService service; + + @Before + public void setup() { + MockRoot root = new MockRoot(); + producer = new SimpleConfigProducer<>(root, "test"); + service = new TestService(root, "service"); + serviceList = new ArrayList<>(); + serviceList.add(service); + ConfigDefinitionKey key = new ConfigDefinitionKey("myname", "mynamespace"); + def = new ConfigDefinition("myname", "1", "mynamespace"); + builder = new ConfigPayloadBuilder(def, new ArrayList<String>()); + Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap = new HashMap<>(); + builderMap.put(key, builder); + UserConfigRepo testRepo = new UserConfigRepo(builderMap); + producer.setUserConfigs(testRepo); + } + + @Test + public void require_that_simple_file_fields_are_modified() { + def.addFileDef("fileVal"); + def.addStringDef("stringVal"); + builder.setField("fileVal", "foo.txt"); + builder.setField("stringVal", "foo.txt"); + service.pathToRef.put("foo.txt", new FileNode("fooshash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getObject("fileVal").getValue(), is("fooshash")); + assertThat(builder.getObject("stringVal").getValue(), is("foo.txt")); + } + + @Test + public void require_that_simple_path_fields_are_modified() { + def.addPathDef("fileVal"); + def.addStringDef("stringVal"); + builder.setField("fileVal", "foo.txt"); + builder.setField("stringVal", "foo.txt"); + service.pathToRef.put("foo.txt", new FileNode("fooshash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getObject("fileVal").getValue(), is("fooshash")); + assertThat(builder.getObject("stringVal").getValue(), is("foo.txt")); + } + + @Test + public void require_that_fields_in_inner_arrays_are_modified() { + def.innerArrayDef("inner").addFileDef("fileVal"); + def.innerArrayDef("inner").addStringDef("stringVal"); + ConfigPayloadBuilder inner = builder.getArray("inner").append(); + inner.setField("fileVal", "bar.txt"); + inner.setField("stringVal", "bar.txt"); + service.pathToRef.put("bar.txt", new FileNode("barhash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getArray("inner").get(0).getObject("fileVal").getValue(), is("barhash")); + assertThat(builder.getArray("inner").get(0).getObject("stringVal").getValue(), is("bar.txt")); + } + + @Test + public void require_that_arrays_are_modified() { + def.arrayDef("fileArray").setTypeSpec(new ConfigDefinition.TypeSpec("fileArray", "file", null, null, null, null)); + def.arrayDef("pathArray").setTypeSpec(new ConfigDefinition.TypeSpec("pathArray", "path", null, null, null, null)); + def.arrayDef("stringArray").setTypeSpec(new ConfigDefinition.TypeSpec("stringArray", "string", null, null, null, null)); + builder.getArray("fileArray").append("foo.txt"); + builder.getArray("fileArray").append("bar.txt"); + builder.getArray("pathArray").append("path.txt"); + builder.getArray("stringArray").append("foo.txt"); + service.pathToRef.put("foo.txt", new FileNode("foohash").value()); + service.pathToRef.put("bar.txt", new FileNode("barhash").value()); + service.pathToRef.put("path.txt", new FileNode("pathhash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getArray("fileArray").get(0).getValue(), is("foohash")); + assertThat(builder.getArray("fileArray").get(1).getValue(), is("barhash")); + assertThat(builder.getArray("pathArray").get(0).getValue(), is("pathhash")); + assertThat(builder.getArray("stringArray").get(0).getValue(), is("foo.txt")); + } + + @Test + public void require_that_structs_are_modified() { + def.structDef("struct").addFileDef("fileVal"); + def.structDef("struct").addStringDef("stringVal"); + builder.getObject("struct").setField("fileVal", "foo.txt"); + builder.getObject("struct").setField("stringVal", "foo.txt"); + service.pathToRef.put("foo.txt", new FileNode("foohash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getObject("struct").getObject("fileVal").getValue(), is("foohash")); + assertThat(builder.getObject("struct").getObject("stringVal").getValue(), is("foo.txt")); + } + + @Test + public void require_that_leaf_maps_are_modified() { + def.leafMapDef("fileMap").setTypeSpec(new ConfigDefinition.TypeSpec("fileMap", "file", null, null, null, null)); + def.leafMapDef("pathMap").setTypeSpec(new ConfigDefinition.TypeSpec("pathMap", "path", null, null, null, null)); + def.leafMapDef("stringMap").setTypeSpec(new ConfigDefinition.TypeSpec("stringMap", "string", null, null, null, null)); + builder.getMap("fileMap").put("foo", "foo.txt"); + builder.getMap("fileMap").put("bar", "bar.txt"); + builder.getMap("pathMap").put("path", "path.txt"); + builder.getMap("stringMap").put("bar", "bar.txt"); + service.pathToRef.put("foo.txt", new FileNode("foohash").value()); + service.pathToRef.put("bar.txt", new FileNode("barhash").value()); + service.pathToRef.put("path.txt", new FileNode("pathhash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getMap("fileMap").get("foo").getValue(), is("foohash")); + assertThat(builder.getMap("fileMap").get("bar").getValue(), is("barhash")); + assertThat(builder.getMap("pathMap").get("path").getValue(), is("pathhash")); + assertThat(builder.getMap("stringMap").get("bar").getValue(), is("bar.txt")); + } + + @Test + public void require_that_fields_in_inner_maps_are_modified() { + def.structMapDef("inner").addFileDef("fileVal"); + def.structMapDef("inner").addStringDef("stringVal"); + ConfigPayloadBuilder inner = builder.getMap("inner").put("foo"); + inner.setField("fileVal", "bar.txt"); + inner.setField("stringVal", "bar.txt"); + service.pathToRef.put("bar.txt", new FileNode("barhash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + assertThat(builder.getMap("inner").get("foo").getObject("fileVal").getValue(), is("barhash")); + assertThat(builder.getMap("inner").get("foo").getObject("stringVal").getValue(), is("bar.txt")); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_null_files_are_not_sent() { + def.addFileDef("fileVal"); + service.pathToRef.put("foo.txt", new FileNode("fooshash").value()); + FileSender.sendUserConfiguredFiles(producer, serviceList, new BaseDeployLogger()); + } + + + private static class TestService extends AbstractService { + public Map<String, FileReference> pathToRef = new HashMap<>(); + public TestService(AbstractConfigProducer<?> parent, String name) { + super(parent, name); + } + + @Override + public FileReference sendFile(String relativePath) { + return pathToRef.get(relativePath); + } + + @Override + public int getPortCount() { + return 0; + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java new file mode 100644 index 00000000000..55ed49ed97b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/internal/ReflectionUtilTest.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils.internal; + +import com.yahoo.test.ArraytypesConfig; +import com.yahoo.config.ChangesRequiringRestart; +import com.yahoo.config.ConfigInstance; +import com.yahoo.test.SimpletypesConfig; +import com.yahoo.vespa.config.ConfigKey; +import org.junit.Test; + +import java.util.Set; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @author bjorncs + * @since 5.1 + */ +public class ReflectionUtilTest { + + private static interface ComplexInterface extends SimpletypesConfig.Producer, ArraytypesConfig.Producer { + } + + private static class SimpleProducer implements SimpletypesConfig.Producer { + @Override + public void getConfig(SimpletypesConfig.Builder builder) { + } + } + + private static class ComplexProducer implements ComplexInterface { + @Override + public void getConfig(ArraytypesConfig.Builder builder) { + } + @Override + public void getConfig(SimpletypesConfig.Builder builder) { + } + } + + private static class RestartConfig extends ConfigInstance { + @SuppressWarnings("UnusedDeclaration") + private static boolean containsFieldsFlaggedWithRestart() { + return true; + } + + @SuppressWarnings("UnusedDeclaration") + private ChangesRequiringRestart getChangesRequiringRestart(RestartConfig newConfig) { + return new ChangesRequiringRestart("testing"); + } + } + + private static class NonRestartConfig extends ConfigInstance {} + + @Test + public void requireThatConfigsProducedByInterfaceTakesParentIntoAccount() { + Set<ConfigKey<?>> configs = ReflectionUtil.configsProducedByInterface(ComplexProducer.class, "foo"); + assertThat(configs.size(), is(2)); + assertTrue(configs.contains(new ConfigKey<>(SimpletypesConfig.CONFIG_DEF_NAME, "foo", SimpletypesConfig.CONFIG_DEF_NAMESPACE))); + assertTrue(configs.contains(new ConfigKey<>(ArraytypesConfig.CONFIG_DEF_NAME, "foo", ArraytypesConfig.CONFIG_DEF_NAMESPACE))); + } + + @Test + public void requireThatConfigsProducedByInterfaceAreFound() { + Set<ConfigKey<?>> configs = ReflectionUtil.configsProducedByInterface(SimpleProducer.class, "foo"); + assertThat(configs.size(), is(1)); + assertTrue(configs.contains(new ConfigKey<>(SimpletypesConfig.CONFIG_DEF_NAME, "foo", SimpletypesConfig.CONFIG_DEF_NAMESPACE))); + } + + @Test + public void requireThatRestartMethodsAreDetectedProperly() { + assertFalse(ReflectionUtil.hasRestartMethods(NonRestartConfig.class)); + assertTrue(ReflectionUtil.hasRestartMethods(RestartConfig.class)); + } + + @Test + public void requireThatRestartMethodsAreProperlyInvoked() { + assertTrue(ReflectionUtil.containsFieldsFlaggedWithRestart(RestartConfig.class)); + assertEquals("testing", ReflectionUtil.getChangesRequiringRestart(new RestartConfig(), new RestartConfig()).getName()); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatGetChangesRequiringRestartValidatesParameterTypes() { + ReflectionUtil.getChangesRequiringRestart(new RestartConfig(), new NonRestartConfig()); + } + + +} |