diff options
599 files changed, 14718 insertions, 8287 deletions
diff --git a/README.md b/README.md index 89bd7d95e59..d84ebbe4456 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ You can also setup CentOS 7 natively and install the following build dependencie ### Build Java modules export MAVEN_OPTS="-Xms128m -Xmx1024m" - source /opt/rh/rh-maven33/enable + source /opt/rh/rh-maven35/enable bash bootstrap.sh java mvn -T <num-threads> install diff --git a/application/pom.xml b/application/pom.xml index ac9ba96cb27..db298451a8b 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,10 +20,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> @@ -76,11 +72,6 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> <!-- All dependencies that should be visible in test classpath, but not compile classpath, for user projects must be added in compile scope here. @@ -167,19 +158,6 @@ </configuration> </plugin> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>scala-test-compile</id> - <phase>process-test-resources</phase> - <goals> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index ed0c29a3917..5554ae6a159 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -15,7 +15,6 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.container.Container; import com.yahoo.container.standalone.StandaloneContainerApplication; -import com.yahoo.container.standalone.StandaloneContainerApplication$; import com.yahoo.docproc.jdisc.DocumentProcessingHandler; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.handler.RequestHandler; @@ -58,10 +57,10 @@ public final class JDisc implements AutoCloseable { return new AbstractModule() { @Override protected void configure() { - bind(Path.class).annotatedWith(StandaloneContainerApplication.applicationPathName()).toInstance(path); - bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.configModelRepoName()).toInstance(configModelRepo); + bind(Path.class).annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME).toInstance(path); + bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.CONFIG_MODEL_REPO_NAME).toInstance(configModelRepo); bind(Boolean.class).annotatedWith( // below is an ugly hack to access fields from a scala object. - Names.named(StandaloneContainerApplication$.MODULE$.disableNetworkingAnnotation())).toInstance( + Names.named(StandaloneContainerApplication.DISABLE_NETWORKING_ANNOTATION)).toInstance( networking == Networking.disable); } }; diff --git a/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java new file mode 100644 index 00000000000..42f3a7267d8 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java @@ -0,0 +1,81 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.yahoo.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.nio.file.Files; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ApplicationBuilderTest { + @Test + public void query_profile_types_can_be_added() throws Exception { + withApplicationBuilder(builder -> { + builder.queryProfileType("MyProfileType", "<query-profile-type id=\"MyProfileType\">" + // + "<field name=\"age\" type=\"integer\" />" + // + "<field name=\"profession\" type=\"string\" />" + // + "<field name=\"user\" type=\"query-profile:MyUserProfile\" />" + // + "</query-profile-type>"); + + assertTrue(Files.exists(builder.getPath().resolve("search/query-profiles/types/MyProfileType.xml"))); + }); + } + + @Test + public void query_profile_can_be_added() throws Exception { + withApplicationBuilder(builder -> { + builder.queryProfile("MyProfile", "<query-profile id=\"MyProfile\">" + // + "<field name=\"message\">Hello world!</field>" + // + "</query-profile>"); + + assertTrue(Files.exists(builder.getPath().resolve("search/query-profiles/MyProfile.xml"))); + }); + } + + @Test + public void rank_expression_can_be_added() throws Exception { + withApplicationBuilder(builder -> { + builder.rankExpression("myExpression", "content"); + assertTrue(Files.exists(builder.getPath().resolve("searchdefinitions/myExpression.expression"))); + }); + } + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + @SuppressWarnings("try") // application unreferenced inside try + public void builder_cannot_be_reused() throws Exception { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(containsString("build method")); + + ApplicationBuilder builder = new ApplicationBuilder(); + builder.servicesXml("<jdisc version=\"1.0\" />"); + try (Application application = builder.build()) { + // do nothing + } + + builder.servicesXml(""); // should fail + } + + private interface TestCase { + void accept(ApplicationBuilder ab) throws Exception; + } + + private static void withApplicationBuilder(TestCase f) throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + try { + f.accept(builder); + } finally { + IOUtils.recursiveDeleteDir(builder.getPath().toFile()); + } + } +} diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index 9388c8f400e..7b4f39b6ac4 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -372,4 +372,21 @@ public class ApplicationTest { "</jdisc>"; } + @Test + public void application_with_access_control_can_be_constructed() throws Exception { + try (Application application = Application.fromServicesXml(servicesXmlWithAccessControl(), Networking.disable)) { + Application unused = application; + } + } + + private static String servicesXmlWithAccessControl() { + return "<jdisc version='1.0'>" + + " <http> <server port='" + 0 +"' id='foo'/> " + + " <filtering>" + + " <access-control domain='foo' />" + + " </filtering>" + + " </http>" + + "</jdisc>"; + } + } diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java new file mode 100644 index 00000000000..729439e7e5a --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java @@ -0,0 +1,60 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.searchers.AddHitSearcher; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @author ollivir + */ +public class JDiscContainerSearchTest { + @Test + public void processing_and_rendering_works() throws Exception { + final String searcherId = AddHitSearcher.class.getName(); + + try (JDisc container = containerWithSearch(searcherId)) { + byte[] rendered = container.search().processAndRender(ComponentSpecification.fromString("mychain"), + ComponentSpecification.fromString("DefaultRenderer"), new Query("")); + String renderedAsString = new String(rendered, "utf-8"); + assertThat(renderedAsString, containsString(searcherId)); + } + } + + @Test + public void searching_works() throws Exception { + final String searcherId = AddHitSearcher.class.getName(); + + try (JDisc container = containerWithSearch(searcherId)) { + Search searching = container.search(); + Result result = searching.process(ComponentSpecification.fromString("mychain"), new Query("")); + String hitTitle = result.hits().get(0).getField("title").toString(); + assertThat(hitTitle, is(searcherId)); + } + } + + public JDisc containerWithSearch(String searcherId) { + return JDisc.fromServicesXml("<container version=\"1.0\">" + // + "<search>" + // + "<chain id=\"mychain\">" + // + "<searcher id=\"" + searcherId + "\"/>" + // + "</chain>" + // + "</search>" + // + "</container>", Networking.disable); + } + + @Test(expected = UnsupportedOperationException.class) + public void retrieving_search_from_container_without_search_is_illegal() throws Exception { + try (JDisc container = JDisc.fromServicesXml("<container version=\"1.0\" />", Networking.disable)) { + container.search(); // throws + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/JDiscTest.java b/application/src/test/java/com/yahoo/application/container/JDiscTest.java new file mode 100644 index 00000000000..7963c2dd165 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/JDiscTest.java @@ -0,0 +1,179 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.yahoo.application.Application; +import com.yahoo.application.ApplicationBuilder; +import com.yahoo.application.Networking; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.application.container.handlers.TestHandler; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.Container; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; +import com.yahoo.jdisc.service.ServerProvider; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import org.junit.Ignore; +import org.junit.Test; + +import java.nio.charset.CharacterCodingException; +import java.nio.file.FileSystems; + +import static com.yahoo.application.container.JDisc.fromServicesXml; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class JDiscTest { + @Test + public void jdisc_can_be_used_as_top_level_element() throws Exception { + try (JDisc container = fromServicesXml("<jdisc version=\"1.0\">" + // + "<search />" + // + "</jdisc>", Networking.disable)) { + assertNotNull(container.search()); + } + } + + @Test + public void jdisc_id_can_be_set() throws Exception { + try (JDisc container = fromServicesXml("<jdisc version=\"1.0\" id=\"my-service-id\">" + // + "<search />" + // + "</jdisc>", Networking.disable)) { + assertNotNull(container.search()); + } + } + + @Test + public void jdisc_can_be_embedded_in_services_tag() throws Exception { + try (JDisc container = fromServicesXml("<services>" + // + "<jdisc version=\"1.0\" id=\"my-service-id\">" + // + "<search />" + // + "</jdisc>" + // + "</services>", Networking.disable)) { + assertNotNull(container.search()); + } + } + + @Test + @SuppressWarnings("try") // container is unused inside the try block + public void multiple_jdisc_elements_gives_exception() { + try (JDisc container = fromServicesXml("<services>" + // + "<jdisc version=\"1.0\" id=\"id1\" />" + // + "<jdisc version=\"1.0\" />" + // + "<container version=\"1.0\"/>" + // + "</services>", Networking.disable)) { + fail("expected exception"); + } catch (Exception e) { + assertThat(e.getMessage(), containsString("container id='', jdisc id='id1', jdisc id=''")); + } + } + + @Test + public void handleRequest_yields_response_from_correct_request_handler() throws Exception { + final String handlerClass = TestHandler.class.getName(); + try (JDisc container = fromServicesXml("<container version=\"1.0\">" + // + "<handler id=\"test-handler\" class=\"" + handlerClass + "\">" + // + "<binding>http://*/TestHandler</binding>" + // + "</handler>" + // + "</container>", Networking.disable)) { + Response response = container.handleRequest(new Request("http://foo/TestHandler")); + try { + assertThat(response.getBodyAsString(), is(TestHandler.RESPONSE)); + } catch (CharacterCodingException e) { + throw new RuntimeException(e); + } + } + } + + @Test + public void load_searcher_from_bundle() throws Exception { + try (JDisc container = JDisc.fromPath(FileSystems.getDefault().getPath("src/test/app-packages/searcher-app"), + Networking.disable)) { + Result result = container.search().process(ComponentSpecification.fromString("default"), + new Query("?query=ignored")); + assertThat(result.hits().get(0).getField("title").toString(), is("Heal the World!")); + } + } + + @Test + public void document_types_can_be_accessed() throws Exception { + try (Application application = new ApplicationBuilder().documentType("example", EXAMPLE_DOCUMENT) + .servicesXml(CONTAINER_WITH_DOCUMENT_PROCESSING).build()) { + JDisc container = application.getJDisc("jdisc"); + DocumentProcessing processing = container.documentProcessing(); + assertThat(processing.getDocumentTypes().keySet(), hasItem("example")); + } + } + + @Test + public void annotation_types_can_be_accessed() throws Exception { + try (Application application = new ApplicationBuilder().documentType("example", "search example {\n" + // + " " + EXAMPLE_DOCUMENT + "\n" + // + " annotation exampleAnnotation {}\n" + // + "}\n").// + servicesXml(CONTAINER_WITH_DOCUMENT_PROCESSING).build()) { + JDisc container = application.getJDisc("jdisc"); + DocumentProcessing processing = container.documentProcessing(); + assertThat(processing.getAnnotationTypes().keySet(), hasItem("exampleAnnotation")); + } + } + + @Ignore // Enable this when static state has been removed. + @Test + public void multiple_containers_can_be_run_in_parallel() throws Exception { + try (JDisc jdisc1 = jdiscWithHttp(); JDisc jdisc2 = jdiscWithHttp()) { + sendRequest(jdisc1); + sendRequest(jdisc2); + } + } + + private void sendRequest(JDisc jdisc) throws CharacterCodingException { + Response response = jdisc.handleRequest(new Request("http://foo/TestHandler")); + assertThat(response.getBodyAsString(), is(TestHandler.RESPONSE)); + } + + public static final String CONTAINER_WITH_DOCUMENT_PROCESSING = // + "<jdisc version=\"1.0\">" + // + "<http />" + // + "<document-processing />" + // + "</jdisc>"; + + public static final String EXAMPLE_DOCUMENT = // + "document example {\n" + // + "\n" + // + " field title type string {\n" + // + " indexing: summary | index # How this field should be indexed\n" + // + " weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature\n" + // + " header\n" + // + " }\n" + // + "}\n"; + + protected JDisc jdiscWithHttp() { + final String handlerId = TestHandler.class.getName(); + final String xml = // + "<jdisc version=\"1.0\">" + // + "<handler id=" + handlerId + " />" + // + "<http>\n" + // + "<server id=\"main\" port=\"9999\" />\n" + // + "</http>\n" + // + "</jdisc>"; + return JDisc.fromServicesXml(xml, Networking.disable); + } + + public static int getListenPort() { + for (ServerProvider server : Container.get().getServerProviderRegistry().allComponents()) { + if (null != server && server instanceof JettyHttpServer) { + return ((JettyHttpServer) server).getListenPort(); + } + } + throw new RuntimeException("No http server found"); + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/TestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/TestHandler.java new file mode 100644 index 00000000000..c31df43bf72 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/TestHandler.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.FastContentWriter; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; + +/** + * @author gjoranv + * @author ollivir + */ +public class TestHandler extends AbstractRequestHandler { + public static final String RESPONSE = "Hello, World!"; + + public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { + FastContentWriter writer = ResponseDispatch.newInstance(com.yahoo.jdisc.Response.Status.OK) + .connectFastWriter(handler); + writer.write(RESPONSE); + writer.close(); + return null; + } + +} diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java new file mode 100644 index 00000000000..9c3cd1e612c --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java @@ -0,0 +1,194 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.JDiscTest; +import com.yahoo.application.container.jersey.resources.TestResource; +import com.yahoo.application.container.jersey.resources.nestedpackage1.NestedTestResource1; +import com.yahoo.application.container.jersey.resources.nestedpackage2.NestedTestResource2; +import com.yahoo.container.Container; +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; +import com.yahoo.osgi.maven.ProjectBundleClassPaths; +import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.Test; + +import javax.ws.rs.core.UriBuilder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class JerseyTest { + private final Path testJar = Paths.get("target/test-jars/jersey-resources.jar"); + private final String testClassesDirectory = "target/test-classes"; + private final String bundleSymbolicName = "myBundle"; + + private final Set<Class<? extends TestResourceBase>> classPathResources; + private final Set<Class<? extends TestResourceBase>> jarFileResources; + + public JerseyTest() { + classPathResources = new HashSet<>(); + classPathResources.add(TestResource.class); + classPathResources.add(NestedTestResource1.class); + classPathResources.add(NestedTestResource2.class); + + jarFileResources = new HashSet<>(); + jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.TestResource.class); + jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.nestedpackage1.NestedTestResource1.class); + jarFileResources.add(com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2.class); + } + + @Test + public void jersey_resources_on_classpath_can_be_invoked_from_application() throws Exception { + saveMainBundleClassPathMappings(testClassesDirectory); + + with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); + } + + @Test + public void jersey_resources_in_provided_dependencies_can_be_invoked_from_application() throws Exception { + BundleClasspathMapping providedDependency = new BundleClasspathMapping(bundleSymbolicName, + Arrays.asList(testClassesDirectory)); + + save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), + Arrays.asList(providedDependency))); + with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); + } + + @Test + public void jersey_resource_on_classpath_can_be_filtered_using_packages() throws Exception { + saveMainBundleClassPathMappings(testClassesDirectory); + + with_jersey_resources(Arrays.asList("com.yahoo.application.container.jersey.resources", + "com.yahoo.application.container.jersey.resources.nestedpackage1"), httpGetter -> { + Class<NestedTestResource2> nestedResource2 = NestedTestResource2.class; + assertDoesNotRespond(nestedResource2, httpGetter); + assertResourcesResponds(copySetExcept(classPathResources, nestedResource2), httpGetter); + }); + } + + @Test + public void jersey_resource_in_jar_can_be_invoked_from_application() throws Exception { + saveMainBundleJarClassPathMappings(testJar); + + with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(jarFileResources, httpGetter)); + } + + @Test + public void jersey_resource_in_jar_can_be_filtered_using_packages() throws Exception { + saveMainBundleJarClassPathMappings(testJar); + + with_jersey_resources(Arrays.asList("com.yahoo.container.test.jars.jersey.resources", + "com.yahoo.container.test.jars.jersey.resources.nestedpackage1"), httpGetter -> { + Class<com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2> nestedResource2 = com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2.class; + + assertDoesNotRespond(nestedResource2, httpGetter); + assertResourcesResponds(copySetExcept(jarFileResources, nestedResource2), httpGetter); + }); + } + + private static <T> Set<T> copySetExcept(Set<T> in, T except) { + Set<T> ret = new HashSet<>(in); + ret.remove(except); + return ret; + } + + private interface ThrowingConsumer<T> { + public void accept(T arg) throws Exception; + } + + private interface HttpGetter { + public HttpResponse get(String path) throws Exception; + } + + @SuppressWarnings("try") // jdisc unreferenced inside try + private void with_jersey_resources(List<String> packagesToScan, ThrowingConsumer<HttpGetter> f) throws Exception { + StringBuffer packageElements = new StringBuffer(); + for (String p : packagesToScan) { + packageElements.append("<package>"); + packageElements.append(p); + packageElements.append("</package>"); + } + + try (JDisc jdisc = JDisc.fromServicesXml( + "<services>" + // + "<jdisc version=\"1.0\" id=\"default\" jetty=\"true\">" + // + "<rest-api path=\"rest-api\" jersey2=\"true\">" + // + "<components bundle=\"" + bundleSymbolicName + "\">" + // + packageElements + // + "</components>" + // + "</rest-api>" + // + "<http>" + // + "<server id=\"mainServer\" port=\"0\" />" + // + "</http>" + // + "</jdisc>" + // + "</services>", // + Networking.enable)) { + final int port = JDiscTest.getListenPort(); + f.accept(path -> { + String p = path.startsWith("/") ? path.substring(1) : path; + CloseableHttpClient client = HttpClientBuilder.create().build(); + return client.execute(new HttpGet("http://localhost:" + port + "/rest-api/" + p)); + }); + } + } + + public void assertResourcesResponds(Collection<Class<? extends TestResourceBase>> resourceClasses, + HttpGetter httpGetter) throws Exception { + for (Class<? extends TestResourceBase> resource : resourceClasses) { + HttpResponse response = httpGetter.get(path(resource)); + assertThat("Failed sending response to " + resource, response.getStatusLine().getStatusCode(), is(200)); + + String content = IOUtils.toString(response.getEntity().getContent()); + assertThat(content, is(TestResourceBase.content(resource))); + } + } + + public void assertDoesNotRespond(Class<? extends TestResourceBase> resourceClass, HttpGetter httpGetter) + throws Exception { + HttpResponse response = httpGetter.get(path(resourceClass)); + assertThat(response.getStatusLine().getStatusCode(), is(404)); + EntityUtils.consume(response.getEntity()); + } + + public void saveMainBundleJarClassPathMappings(Path jarFile) throws Exception { + assertTrue("Couldn't find file " + jarFile + ", please remember to run mvn process-test-resources first.", + Files.isRegularFile(jarFile)); + saveMainBundleClassPathMappings(jarFile.toAbsolutePath().toString()); + } + + public void saveMainBundleClassPathMappings(String classPathElement) throws Exception { + BundleClasspathMapping mainBundleClassPathMappings = new BundleClasspathMapping(bundleSymbolicName, + Arrays.asList(classPathElement)); + save(new ProjectBundleClassPaths(mainBundleClassPathMappings, emptyList())); + } + + public void save(ProjectBundleClassPaths projectBundleClassPaths) throws Exception { + Path path = Paths.get(testClassesDirectory).resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME); + ProjectBundleClassPaths.save(path, projectBundleClassPaths); + } + + public String path(Class<?> resourceClass) { + return UriBuilder.fromResource(resourceClass).build().toString(); + } +} diff --git a/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java b/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java new file mode 100644 index 00000000000..5b6f1fa9c35 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/jersey/resources/TestResource.java @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources; + +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; + +import javax.ws.rs.Path; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +@Path("/test-resource") +public class TestResource extends TestResourceBase { +} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.scala b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.java index c50d7cd6f57..d4901995152 100644 --- a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.scala +++ b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.java @@ -1,12 +1,14 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources.nestedpackage1 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources.nestedpackage1; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes + * @author ollivir */ @Path("/nested-test-resource1") -class NestedTestResource1 extends TestResourceBase +public class NestedTestResource1 extends TestResourceBase { +} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.scala b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.java index 50f0054b6ec..1763023a533 100644 --- a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.scala +++ b/application/src/test/java/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.java @@ -1,12 +1,14 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources.nestedpackage2 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources.nestedpackage2; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes + * @author ollivir */ @Path("/nested-test-resource2") -class NestedTestResource2 extends TestResourceBase +public class NestedTestResource2 extends TestResourceBase { +} diff --git a/application/src/test/java/com/yahoo/application/container/searchers/AddHitSearcher.java b/application/src/test/java/com/yahoo/application/container/searchers/AddHitSearcher.java new file mode 100644 index 00000000000..274c37bcfc1 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/searchers/AddHitSearcher.java @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.searchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +public class AddHitSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(getDummyHit()); + + return result; + } + + private Hit getDummyHit() { + Hit hit = new Hit("dummy"); + hit.setField("title", getId().getName()); + return hit; + } +} diff --git a/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala b/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala deleted file mode 100644 index 255810b959b..00000000000 --- a/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application - -import container.JDiscTest._ -import java.nio.file.Files -import org.junit.Assert.{assertTrue, assertThat, fail} -import org.junit.Test -import com.yahoo.io.IOUtils -import org.hamcrest.CoreMatchers.containsString - -/** - * @author tonytv - */ - class ApplicationBuilderTest { - @Test - def query_profile_types_can_be_added() { - withApplicationBuilder { builder => - builder.queryProfileType("MyProfileType", - <query-profile-type id="MyProfileType"> - <field name="age" type="integer" /> - <field name="profession" type="string" /> - <field name="user" type="query-profile:MyUserProfile" /> - </query-profile-type>) - - assertTrue(Files.exists(builder.getPath.resolve("search/query-profiles/types/MyProfileType.xml"))) - } - } - - - @Test - def query_profile_can_be_added() { - withApplicationBuilder { builder => - builder.queryProfile("MyProfile", - <query-profile id="MyProfile"> - <field name="message">Hello world!</field> - </query-profile>) - - assertTrue(Files.exists(builder.getPath.resolve("search/query-profiles/MyProfile.xml"))) - } - } - - @Test - def rank_expression_can_be_added() { - withApplicationBuilder { builder => - builder.rankExpression("myExpression", "content") - assertTrue(Files.exists(builder.getPath.resolve("searchdefinitions/myExpression.expression"))) - } - } - - @Test - def builder_cannot_be_reused() { - val builder = new ApplicationBuilder - builder.servicesXml(<jdisc version="1.0" />) - - using(builder.build()) { builder => } - - try { - builder.servicesXml("") - fail("Expected exception.") - } catch { - case e: RuntimeException => assertThat(e.getMessage, containsString("build method")) - } - - } - - def withApplicationBuilder(f: ApplicationBuilder => Unit) { - val builder = new ApplicationBuilder() - try { - f(builder) - } finally { - IOUtils.recursiveDeleteDir(builder.getPath.toFile) - } - } -} diff --git a/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala b/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala deleted file mode 100644 index 9d7e8ea26ab..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container - -import org.junit.Test -import searchers.AddHitSearcher -import com.yahoo.component.ComponentSpecification -import com.yahoo.search.Query -import org.junit.Assert._ -import org.hamcrest.CoreMatchers._ -import org.hamcrest.Matchers.containsString; -import JDiscTest.{fromServicesXml, using} - - -/** - * - * @author gjoranv - * @since 5.1.15 - */ -class JDiscContainerSearchTest { - - @Test - def processing_and_rendering_works() { - val searcherId = classOf[AddHitSearcher].getName - - using(containerWithSearch(searcherId)) - { container => - val rendered = container.search.processAndRender(ComponentSpecification.fromString("mychain"), - ComponentSpecification.fromString("DefaultRenderer"), new Query("")) - val renderedAsString = new String(rendered, "utf-8") - assertThat(renderedAsString, containsString(searcherId)) - } - } - - @Test - def searching_works() { - val searcherId = classOf[AddHitSearcher].getName - - using(containerWithSearch(searcherId)) - { container => - val searching = container.search - val result = searching.process(ComponentSpecification.fromString("mychain"), new Query("")) - - val hitTitle = result.hits().get(0).getField("title").asInstanceOf[String] - assertThat(hitTitle, is(searcherId)) - } - } - - def containerWithSearch(searcherId: String) = { - fromServicesXml( - <container version="1.0"> - <search> - <chain id="mychain"> - <searcher id={searcherId}/> - </chain> - </search> - </container>) - } - - @Test(expected = classOf[UnsupportedOperationException]) - def retrieving_search_from_container_without_search_is_illegal() { - using(JDiscTest.fromServicesXml( - <container version="1.0" /> - )) - { container => - container.search // throws - } - } - -} diff --git a/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala b/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala deleted file mode 100644 index 727a77ee119..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container - -import com.yahoo.container.Container -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer - -import scala.language.implicitConversions -import handler.Request -import org.junit.{Ignore, Test} -import org.junit.Assert.{assertThat, assertNotNull, fail} -import org.hamcrest.CoreMatchers.{is, containsString, hasItem} -import java.nio.file.FileSystems -import com.yahoo.search.Query -import com.yahoo.component.ComponentSpecification -import handlers.TestHandler -import xml.{Node, Elem} -import JDiscTest._ -import com.yahoo.application.{Networking, ApplicationBuilder} - -import scala.collection.convert.wrapAsScala._ - - -/** - * @author tonytv - * @author gjoranv - */ -class JDiscTest { - @Test - def jdisc_can_be_used_as_top_level_element() { - using(fromServicesXml( - <jdisc version="1.0"> - <search /> - </jdisc>)) - { container => - assertNotNull(container.search()) - } - } - - @Test - def jdisc_id_can_be_set() { - using(fromServicesXml( - <jdisc version="1.0" id="my-service-id"> - <search /> - </jdisc>)) - { container => - assertNotNull(container.search()) - } - } - - @Test - def jdisc_can_be_embedded_in_services_tag() { - using(fromServicesXml( - <services> - <jdisc version="1.0" id="my-service-id"> - <search /> - </jdisc> - </services>)) - { container => - assertNotNull(container.search()) - } - } - - @Test - def multiple_jdisc_elements_gives_exception() { - try { - using(fromServicesXml( - <services> - <jdisc version="1.0" id="id1" /> - <jdisc version="1.0" /> - <container version="1.0"/> - </services>)) - { container => fail("expected exception")} - } catch { - case e: Exception => assertThat(e.getMessage, containsString("container id='', jdisc id='id1', jdisc id=''")) - } - } - - @Test - def handleRequest_yields_response_from_correct_request_handler() { - using(fromServicesXml( - <container version="1.0"> - <handler id="test-handler" class={classOf[TestHandler].getName}> - <binding>http://*/TestHandler</binding> - </handler> - </container>)) - { container => - val response = container.handleRequest(new Request("http://foo/TestHandler")) - assertThat(response.getBodyAsString, is(TestHandler.RESPONSE)) - } - } - - @Test - def load_searcher_from_bundle() { - using(JDisc.fromPath(FileSystems.getDefault.getPath("src/test/app-packages/searcher-app"), Networking.disable)) - { container => - val result = container.search.process(ComponentSpecification.fromString("default"),new Query("?query=ignored")) - assertThat(result.hits().get(0).getField("title").toString, is("Heal the World!")) - } - } - - @Test - def document_types_can_be_accessed() { - using(new ApplicationBuilder().documentType("example", exampleDocument). - servicesXml(containerWithDocumentProcessing). - build()) - { application => - val container = application.getJDisc("jdisc") - val processing = container.documentProcessing() - assertThat(processing.getDocumentTypes.keySet(), hasItem("example")) - } - } - - @Test - def annotation_types_can_be_accessed() { - using(new ApplicationBuilder().documentType("example", - s""" - |search example { - | ${exampleDocument} - | annotation exampleAnnotation {} - |} - """.stripMargin). - servicesXml(containerWithDocumentProcessing). - build()) - { application => - val container = application.getJDisc("jdisc") - val processing = container.documentProcessing() - assertThat(processing.getAnnotationTypes.keySet(), hasItem("exampleAnnotation")) - } - } - - @Ignore //Enable this when static state has been removed. - @Test - def multiple_containers_can_be_run_in_parallel() { - def sendRequest(jdisc: JDisc) { - val response = jdisc.handleRequest(new Request("http://foo/TestHandler")) - assertThat(response.getBodyAsString, is(TestHandler.RESPONSE)) - } - - using(jdiscWithHttp()) { jdisc1 => - using(jdiscWithHttp()) { jdisc2 => - sendRequest(jdisc1) - sendRequest(jdisc2) - } - } - } -} - -object JDiscTest { - - def fromServicesXml(elem: Elem, networking: Networking = Networking.disable) = - JDisc.fromServicesXml(elem.toString(), networking) - - def using[T <: AutoCloseable, U](t: T)(f: T => U ) = { - try { - f(t) - } finally { - t.close() - } - } - - implicit def xmlToString(xml: Node): String = xml.toString() - - val containerWithDocumentProcessing = - <jdisc version="1.0"> - <http /> - <document-processing /> - </jdisc> - - val exampleDocument = - """ - |document example { - | - | field title type string { - | indexing: summary | index # How this field should be indexed - | weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature - | header - | } - |} - """.stripMargin - - - def jdiscWithHttp() = { - fromServicesXml( - <jdisc version="1.0"> - <handler id={classOf[TestHandler].getName} /> - <http> - <server id="main" port="9999" /> - </http> - </jdisc>) - } - - def getListenPort: Int = - Container.get.getServerProviderRegistry.allComponents().collectFirst { - case server: JettyHttpServer => server.getListenPort - } getOrElse { - throw new RuntimeException("No http server found") - } -} diff --git a/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala b/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala deleted file mode 100644 index 067ba3f31f1..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.handlers - -import com.yahoo.jdisc.handler.{ResponseDispatch, ResponseHandler, AbstractRequestHandler} -import TestHandler._ - -/** - * @author gjoranv - * @since 5.1.15 - */ - -class TestHandler extends AbstractRequestHandler { - def handleRequest(request:JDiscRequest, handler: ResponseHandler) = { - val writer = ResponseDispatch.newInstance(com.yahoo.jdisc.Response.Status.OK).connectFastWriter(handler) - writer.write(RESPONSE) - writer.close() - null - } -} -object TestHandler { - val RESPONSE = "Hello, World!" - type JDiscRequest = com.yahoo.jdisc.Request -} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala b/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala deleted file mode 100644 index 2aee68f254a..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey - -import java.nio.file.{Files, Path, Paths} -import javax.ws.rs.core.UriBuilder - -import com.yahoo.application.Networking -import com.yahoo.application.container.JDiscTest._ -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase -import com.yahoo.container.test.jars.jersey.{resources => jarResources} -import com.yahoo.osgi.maven.ProjectBundleClassPaths -import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping -import org.apache.http.HttpResponse -import org.apache.http.client.methods.HttpGet -import org.apache.http.impl.client.HttpClientBuilder -import org.apache.http.util.EntityUtils -import org.hamcrest.CoreMatchers.is -import org.junit.Assert._ -import org.junit.Test - -import scala.collection.JavaConverters._ -import scala.io.Source - -/** - * @author tonytv - */ -class JerseyTest { - type TestResourceClass = Class[_ <: TestResourceBase] - - val testJar = Paths.get("target/test-jars/jersey-resources.jar") - val testClassesDirectory = "target/test-classes" - val bundleSymbolicName = "myBundle" - - val classPathResources = Set( - classOf[resources.TestResource], - classOf[resources.nestedpackage1.NestedTestResource1], - classOf[resources.nestedpackage2.NestedTestResource2]) - - val jarFileResources = Set( - classOf[jarResources.TestResource], - classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage1.NestedTestResource1], - classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2]) - - @Test - def jersey_resources_on_classpath_can_be_invoked_from_application(): Unit = { - saveMainBundleClassPathMappings(testClassesDirectory) - - with_jersey_resources() { httpGetter => - assertResourcesResponds(classPathResources, httpGetter) - } - } - - @Test - def jersey_resources_in_provided_dependencies_can_be_invoked_from_application(): Unit = { - val providedDependency = new BundleClasspathMapping(bundleSymbolicName, List(testClassesDirectory).asJava) - - save(new ProjectBundleClassPaths( - new BundleClasspathMapping("main", List().asJava), - List(providedDependency).asJava)) - - with_jersey_resources() { httpGetter => - assertResourcesResponds(classPathResources, httpGetter) - } - } - - @Test - def jersey_resource_on_classpath_can_be_filtered_using_packages(): Unit = { - saveMainBundleClassPathMappings(testClassesDirectory) - - with_jersey_resources( - packagesToScan = List( - "com.yahoo.application.container.jersey.resources", - "com.yahoo.application.container.jersey.resources.nestedpackage1")) - { httpGetter => - val nestedResource2 = classOf[resources.nestedpackage2.NestedTestResource2] - - assertDoesNotRespond(nestedResource2, httpGetter) - assertResourcesResponds(classPathResources - nestedResource2, httpGetter) - } - } - - @Test - def jersey_resource_in_jar_can_be_invoked_from_application(): Unit = { - saveMainBundleJarClassPathMappings(testJar) - - with_jersey_resources() { httpGetter => - assertResourcesResponds(jarFileResources, httpGetter) - } - } - - @Test - def jersey_resource_in_jar_can_be_filtered_using_packages(): Unit = { - saveMainBundleJarClassPathMappings(testJar) - - with_jersey_resources( - packagesToScan = List( - "com.yahoo.container.test.jars.jersey.resources", - "com.yahoo.container.test.jars.jersey.resources.nestedpackage1")) - { httpGetter => - val nestedResource2 = classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2] - - assertDoesNotRespond(nestedResource2, httpGetter) - assertResourcesResponds(jarFileResources - nestedResource2, httpGetter) - } - } - - def with_jersey_resources(packagesToScan: List[String] = List())( f: HttpGetter => Unit): Unit = { - val packageElements = packagesToScan.map { p => <package>{p}</package>} - - using(fromServicesXml( - <services> - <jdisc version="1.0" id="default" jetty="true"> - <rest-api path="rest-api" jersey2="true"> - <components bundle={bundleSymbolicName}> - { packageElements } - </components> - </rest-api> - <http> - <server id="mainServer" port="0" /> - </http> - </jdisc> - </services>, - Networking.enable)) { jdisc => - - - def httpGetter(path: HttpPath) = { - val client = HttpClientBuilder.create().build() - client.execute(new HttpGet(s"http://localhost:$getListenPort/rest-api/${path.stripPrefix("/")}")) - } - - f(httpGetter) - } - } - - def assertResourcesResponds(resourceClasses: Traversable[TestResourceClass], httpGetter: HttpGetter): Unit = { - for (resource <- resourceClasses) { - val response = httpGetter(path(resource)) - assertThat(s"Failed sending response to $resource", response.getStatusLine.getStatusCode, is(200)) - - val content = Source.fromInputStream(response.getEntity.getContent).mkString - assertThat(content, is(TestResourceBase.content(resource))) - } - } - - def assertDoesNotRespond(resourceClass: TestResourceClass, httpGetter: HttpGetter): Unit = { - val response = httpGetter(path(resourceClass)) - assertThat(response.getStatusLine.getStatusCode, is(404)) - EntityUtils.consume(response.getEntity) - } - - def saveMainBundleJarClassPathMappings(jarFile: Path): Unit = { - assertTrue(s"Couldn't find file $jarFile, please remember to run mvn process-test-resources first.", Files.isRegularFile(jarFile)) - saveMainBundleClassPathMappings(jarFile.toAbsolutePath.toString) - } - - def saveMainBundleClassPathMappings(classPathElement: String): Unit = { - val mainBundleClassPathMappings = new BundleClasspathMapping(bundleSymbolicName, List(classPathElement).asJava) - save(new ProjectBundleClassPaths(mainBundleClassPathMappings, List().asJava)) - } - - def save(projectBundleClassPaths: ProjectBundleClassPaths): Unit = { - val path = Paths.get(testClassesDirectory).resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME) - ProjectBundleClassPaths.save(path, projectBundleClassPaths) - } - - def path(resourceClass: TestResourceClass) = { - UriBuilder.fromResource(resourceClass).build().toString - } - - type HttpPath = String - type HttpGetter = HttpPath => HttpResponse -} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala b/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala deleted file mode 100644 index 5dc16102d8e..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.jersey.resources - -import javax.ws.rs.Path -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase - - -/** - * @author tonytv - */ -@Path("/test-resource") -class TestResource extends TestResourceBase diff --git a/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala b/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala deleted file mode 100644 index e3a6cd031bc..00000000000 --- a/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.searchers - -import com.yahoo.search.{Searcher, Result, Query} -import com.yahoo.search.searchchain.Execution -import com.yahoo.search.result.Hit - - -class AddHitSearcher extends Searcher { - - override def search(query: Query, execution: Execution) : Result = { - val result = execution.search(query) - result.hits().add(dummyHit) - - result - } - - private def dummyHit = { - val hit = new Hit("dummy") - hit.setField("title", getId.getName) - hit - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 5f330dc01bf..e457df37946 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -29,6 +30,7 @@ import java.util.logging.Logger; */ public class InstanceValidator { + private static final AthenzService TENANT_DOCKER_CONTAINER_IDENTITY = new AthenzService("vespa.vespa.tenant"); private static final Logger log = Logger.getLogger(InstanceValidator.class.getName()); static final String SERVICE_PROPERTIES_DOMAIN_KEY = "identity.domain"; static final String SERVICE_PROPERTIES_SERVICE_KEY = "identity.service"; @@ -81,6 +83,7 @@ public class InstanceValidator { // If/when we dont care about logging exactly whats wrong, this can be simplified boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { + Optional<ApplicationInfo> applicationInfo = superModelProvider.getSuperModel().getApplicationInfo(applicationId); if (!applicationInfo.isPresent()) { @@ -88,6 +91,10 @@ public class InstanceValidator { return false; } + if (TENANT_DOCKER_CONTAINER_IDENTITY.equals(new AthenzService(domain, service))) { + return true; + } + Optional<ServiceInfo> matchingServiceInfo = applicationInfo.get() .getModel() .getHosts() diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java index f48dffecb27..cd13305c009 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -8,27 +8,37 @@ package com.yahoo.config.model.application.provider; */ public class DeployData { - /* Which user deployed */ + /** Which user deployed */ private final String deployedByUser; - /* Name of application given by user */ + /** Name of application given by user */ private final String applicationName; - /* The absolute path to the directory holding the application */ + /** The absolute path to the directory holding the application */ private final String deployedFromDir; - /* Timestamp when a deployment was made */ + /** Timestamp when a deployment was made */ private final long deployTimestamp; + /** Whether this is an internal redeploy, not caused by an application package change */ + private final boolean internalRedeploy; + /* Application generation. Incremented by one each time an application is deployed. */ private final long generation; private final long currentlyActiveGeneration; - public DeployData(String deployedByUser, String deployedFromDir, String applicationName, Long deployTimestamp, Long generation, long currentlyActiveGeneration) { + public DeployData(String deployedByUser, + String deployedFromDir, + String applicationName, + Long deployTimestamp, + boolean internalRedeploy, + Long generation, + long currentlyActiveGeneration) { this.deployedByUser = deployedByUser; this.deployedFromDir = deployedFromDir; this.applicationName = applicationName; this.deployTimestamp = deployTimestamp; + this.internalRedeploy = internalRedeploy; this.generation = generation; this.currentlyActiveGeneration = currentlyActiveGeneration; } @@ -45,6 +55,8 @@ public class DeployData { return deployTimestamp; } + public boolean isInternalRedeploy() { return internalRedeploy; } + public long getGeneration() { return generation; } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 9bfcd4ecb6d..8fc871a1aa9 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -113,9 +113,13 @@ public class FilesApplicationPackage implements ApplicationPackage { } private static ApplicationMetaData metaDataFromDeployData(File appDir, DeployData deployData) { - return new ApplicationMetaData(deployData.getDeployedByUser(), deployData.getDeployedFromDir(), - deployData.getDeployTimestamp(), deployData.getApplicationName(), - computeCheckSum(appDir), deployData.getGeneration(), + return new ApplicationMetaData(deployData.getDeployedByUser(), + deployData.getDeployedFromDir(), + deployData.getDeployTimestamp(), + deployData.isInternalRedeploy(), + deployData.getApplicationName(), + computeCheckSum(appDir), + deployData.getGeneration(), deployData.getCurrentlyActiveGeneration()); } @@ -566,7 +570,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } public static ApplicationMetaData readMetaData(File appDir) { - ApplicationMetaData defaultMetaData = new ApplicationMetaData(appDir, "n/a", "n/a", 0l, "", 0l, 0l); + ApplicationMetaData defaultMetaData = new ApplicationMetaData(appDir, "n/a", "n/a", 0l, false, "", 0l, 0l); File metaFile = new File(appDir, META_FILE_NAME); if (!metaFile.exists()) { return defaultMetaData; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java index e7e65751ee8..783f7361ad5 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -56,20 +56,23 @@ public class SchemaValidators { */ public SchemaValidators(Version vespaVersion, DeployLogger logger) { this.deployLogger = logger; - File schemaDir; + File schemaDir = null; try { schemaDir = saveSchemasFromJar(new File(SchemaValidators.schemaDirBase), vespaVersion); - } catch (IOException e) { - throw new RuntimeException(e); + servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); + hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); + deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); + validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); + containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); + routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } catch (Exception e) { + throw e; + } finally { + if (schemaDir != null) + IOUtils.recursiveDeleteDir(schemaDir); } - - servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); - hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); - deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); - validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); - containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); - routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); - IOUtils.recursiveDeleteDir(schemaDir); } /** @@ -81,19 +84,19 @@ public class SchemaValidators { this(vespaVersion, new BaseDeployLogger()); } - public SchemaValidator servicesXmlValidator() throws IOException { + public SchemaValidator servicesXmlValidator() { return servicesXmlValidator; } - public SchemaValidator hostsXmlValidator() throws IOException { + public SchemaValidator hostsXmlValidator() { return hostsXmlValidator; } - public SchemaValidator deploymentXmlValidator() throws IOException { + public SchemaValidator deploymentXmlValidator() { return deploymentXmlValidator; } - SchemaValidator validationOverridesXmlValidator() throws IOException { + SchemaValidator validationOverridesXmlValidator() { return validationOverridesXmlValidator; } diff --git a/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java b/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java index ebe93f16738..04405839a9b 100644 --- a/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java +++ b/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java @@ -14,22 +14,25 @@ import java.util.Map; public abstract class ConfigInstance extends InnerNode { public interface Builder extends ConfigBuilder { + /** * Dispatches a getConfig() call if this instance's producer is of the right type * @param producer a config producer * @return true if this instance's producer was the correct type, and hence a getConfig call was dispatched */ - public boolean dispatchGetConfig(Producer producer); + boolean dispatchGetConfig(Producer producer); + + String getDefName(); + String getDefNamespace(); + String getDefMd5(); - public String getDefName(); - public String getDefNamespace(); - public String getDefMd5(); } public interface Producer {} private String configMd5 = ""; + @SuppressWarnings("unused") // Used by reflection from ConfigInstanceUtil String configId; /** diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java index 5f0bc275bad..7d455c58b30 100755 --- a/config-lib/src/main/java/com/yahoo/config/FileReference.java +++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java @@ -7,19 +7,20 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * An immutable file reference that can only be created from classes within the same package. * This is to prevent clients from creating arbitrary and invalid file references. * - * @author tonytv + * @author Tony Vaagenes */ public final class FileReference { private final String value; public FileReference(String value) { - this.value = value; + this.value = Objects.requireNonNull(value); } public String value() { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java index 41a0feff5d4..a3769299cf8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java @@ -11,27 +11,32 @@ import java.io.*; * Metadata about an application package. * * @author hmusum - * @since 5.0 */ public class ApplicationMetaData { + private final String deployedByUser; private final String deployedFromDir; private final long deployTimestamp; + private final boolean internalRedeploy; private final long generation; private final long previousActiveGeneration; private final String checkSum; private final String appName; public ApplicationMetaData(File appDir, String deployedByUser, String deployedFromDir, Long deployTimestamp, + boolean internalRedeploy, String checkSum, Long generation, long previousActiveGeneration) { - this(deployedByUser, deployedFromDir, deployTimestamp, appDir.getName(), checkSum, generation, previousActiveGeneration); + this(deployedByUser, deployedFromDir, deployTimestamp, internalRedeploy, + appDir.getName(), checkSum, generation, previousActiveGeneration); } - public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, String applicationName, String checkSum, Long generation, long previousActiveGeneration) { + public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, + String applicationName, String checkSum, Long generation, long previousActiveGeneration) { this.appName = applicationName; this.deployedByUser = deployedByUser; this.deployedFromDir = deployedFromDir; this.deployTimestamp = deployTimestamp; + this.internalRedeploy = internalRedeploy; this.checkSum = checkSum; this.generation = generation; this.previousActiveGeneration = previousActiveGeneration; @@ -88,6 +93,12 @@ public class ApplicationMetaData { } /** + * Returns whether this application generation was produced by a system internal redeployment, + * not an application package change + */ + public boolean isInternalRedeploy() { return internalRedeploy; } + + /** * Returns an md5 hash of the contents of the application package * @return an md5sum of the application package */ @@ -115,7 +126,14 @@ public class ApplicationMetaData { Inspector root = data.get(); Inspector deploy = root.field("deploy"); Inspector app = root.field("application"); - return new ApplicationMetaData(deploy.field("user").asString(), deploy.field("from").asString(), deploy.field("timestamp").asLong(), app.field("name").asString(), app.field("checksum").asString(), app.field("generation").asLong(), app.field("previousActiveGeneration").asLong()); + return new ApplicationMetaData(deploy.field("user").asString(), + deploy.field("from").asString(), + deploy.field("timestamp").asLong(), + booleanField("internalRedeploy", false, deploy), + app.field("name").asString(), + app.field("checksum").asString(), + app.field("generation").asLong(), + app.field("previousActiveGeneration").asLong()); } catch (Exception e) { throw new IllegalArgumentException("Error parsing json metadata", e); } @@ -128,6 +146,7 @@ public class ApplicationMetaData { deploy.setString("user", deployedByUser); deploy.setString("from", deployedFromDir); deploy.setLong("timestamp", deployTimestamp); + deploy.setBool("internalRedeploy", internalRedeploy); Cursor app = meta.setObject("application"); app.setString("name", appName); app.setString("checksum", checkSum); @@ -136,6 +155,12 @@ public class ApplicationMetaData { return slime; } + private static boolean booleanField(String fieldName, boolean defaultValue, Inspector object) { + Inspector value = object.field(fieldName); + if ( ! value.valid()) return defaultValue; + return value.asBool(); + } + public String asJsonString() { Slime slime = getSlime(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index fe6f7da2092..dd54fe11c39 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -224,9 +224,10 @@ public interface ApplicationPackage { /** * Gets the ApplicationMetaData instance for this application package. + * * @return an ApplicationMetaData instance */ - default ApplicationMetaData getMetaData() { return null; } + ApplicationMetaData getMetaData(); default File getFileReference(Path pathRelativeToAppDir) { throw new UnsupportedOperationException("This application package cannot return file references"); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java index c6a72ebb3ff..486db205a4c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java @@ -4,6 +4,7 @@ package com.yahoo.config.model.api; import com.yahoo.config.provision.ApplicationId; public class ApplicationInfo { + private final ApplicationId applicationId; private final long generation; private final Model model; // NOT immutable @@ -23,4 +24,5 @@ public class ApplicationInfo { public Model getModel() { return model; } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index ac1bcfa542a..9b457f49bd2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -23,12 +23,18 @@ public interface FileDistribution { */ void startDownload(String hostName, int port, Set<FileReference> fileReferences); + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static String getDefaultFileDBRoot() { return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"); } + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static File getDefaultFileDBPath() { return new File(getDefaultFileDBRoot()); } + File getFileReferencesDir(); + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index cc17cae337c..97ad67c5279 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; @@ -47,6 +48,10 @@ public interface Model { */ void distributeFiles(FileDistribution fileDistribution); + /** + * The set of files that should be distributed to the hosts in this model. + */ + Set<FileReference> fileReferences(); /** * Gets the allocated hosts for this model. diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 1857cc33d64..3ef9925510c 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -14,30 +14,29 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> + <artifactId>fat-model-dependencies</artifactId> <version>${project.version}</version> - <scope>provided</scope> + <type>pom</type> </dependency> + <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model</artifactId> - <version>${project.version}</version> + <!-- TODO: remove, we probably don't need version 13. --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>13.0.1</version> </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> + <artifactId>config-model-api</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>provided-dependencies</artifactId> + <artifactId>config-provisioning</artifactId> <version>${project.version}</version> + <scope>provided</scope> <exclusions> <exclusion> <groupId>com.google.inject</groupId> @@ -45,6 +44,8 @@ </exclusion> </exclusions> </dependency> + + <!-- TODO: remove all test deps, should not be needed --> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> @@ -60,35 +61,6 @@ <artifactId>guava-testlib</artifactId> <version>17.0</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>13.0.1</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configdefinitions</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-application-package</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configgen</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-bundle</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <groupId>com.yahoo.vespa</groupId> @@ -102,109 +74,9 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>simplemetrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>yolean</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>documentapi</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vdslib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>document</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>linguistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>processing</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>chain</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> @@ -213,50 +85,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search-and-docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>logd</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchcore</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>storage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vsm</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>indexinglanguage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchsummary</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_http_service</artifactId> - <version>${project.version}</version> - </dependency> </dependencies> <build> diff --git a/config-model/pom.xml b/config-model/pom.xml index 9388608fdcb..bab2f37e667 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -276,11 +276,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> @@ -300,11 +295,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> @@ -531,19 +521,6 @@ <updateReleaseInfo>true</updateReleaseInfo> </configuration> </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 8ca0e5c501c..0cfde3c655c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.test; +import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ApplicationFile; @@ -39,6 +40,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; + private final ApplicationMetaData applicationMetaData; protected MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, @@ -52,6 +54,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), asNamedReaderList(queryProfile)); + applicationMetaData = new ApplicationMetaData("user", "dir", 0L, false, "application", "checksum", 0L, 0L); } @Override @@ -133,6 +136,8 @@ public class MockApplicationPackage implements ApplicationPackage { public QueryProfileRegistry getQueryProfiles() { return queryProfileRegistry; } + public ApplicationMetaData getMetaData() { return applicationMetaData; } + @Override public Reader getRankingExpression(String name) { File expressionFile = new File(searchDefinitionDir, name); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 72ba6de7022..fc75f7a19fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -2,8 +2,12 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -41,18 +45,73 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - if (field.usesStructOrMap() && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { - return; // Ignore struct fields for indexed search (only implemented for streaming search) + if (unsupportedFieldType(field)) { + return; // Ignore majority of struct fields for indexed search (only implemented for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); + } else if (isArrayOfSimpleStruct(field)) { + deriveArrayOfSimpleStruct(field); + } else if (isMapOfSimpleStruct(field)) { + deriveMapOfSimpleStruct(field); } else { deriveAttributes(field); } } + private static boolean unsupportedFieldType(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isArrayOfSimpleStruct(field) && + !isMapOfSimpleStruct(field) && + !field.getDataType().equals(PositionDataType.INSTANCE) && + !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + } + + private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)fieldType; + return isSimpleStruct(arrayType.getNestedType()); + } else { + return false; + } + } + + private static boolean isMapOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof MapDataType) { + MapDataType mapType = (MapDataType)fieldType; + return isPrimitiveType(mapType.getKeyType()) && + isSimpleStruct(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isSimpleStruct(DataType type) { + if (type instanceof StructDataType && + !(type.equals(PositionDataType.INSTANCE))) { + StructDataType structType = (StructDataType) type; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; + } + } + return true; + } else { + return false; + } + } + + private static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + /** Returns an attribute by name, or null if it doesn't exist */ public Attribute getAttribute(String attributeName) { return attributes.get(attributeName); @@ -98,6 +157,35 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } } + private void deriveArrayOfSimpleStruct(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + deriveAttributesAsArrayType(structField); + } + } + + private void deriveAttributesAsArrayType(ImmutableSDField field) { + for (Attribute attribute : field.getAttributes().values()) { + if (field.getName().equals(attribute.getName())) { + attributes.put(attribute.getName(), attribute.convertToArray()); + } + } + } + + private void deriveMapOfSimpleStruct(ImmutableSDField field) { + deriveMapKeyField(field.getStructField("key")); + deriveMapValueField(field.getStructField("value")); + } + + private void deriveMapKeyField(ImmutableSDField keyField) { + deriveAttributesAsArrayType(keyField); + } + + private void deriveMapValueField(ImmutableSDField valueField) { + for (ImmutableSDField structField : valueField.getStructFields()) { + deriveAttributesAsArrayType(structField); + } + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index f932265cb93..81e44850e71 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -147,6 +147,12 @@ public final class Attribute implements Cloneable, Serializable { this.referenceDocumentType = referenceDocumentType; } + public Attribute convertToArray() { + Attribute result = clone(); + result.collectionType = CollectionType.ARRAY; + return result; + } + /** * <p>Returns whether this attribute should be included in the "attributeprefetch" summary * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. @@ -181,6 +187,7 @@ public final class Attribute implements Cloneable, Serializable { public long upperBound() { return upperBound; } public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } + public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } public Sorting getSorting() { return sorting; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index 67d60b08ab0..6ca16c1559d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -21,6 +21,7 @@ public class ExpressionTransforms { private final List<ExpressionTransformer> transforms = ImmutableList.of(new TensorFlowFeatureConverter(), + new OnnxFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), new MacroInliner(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java new file mode 100644 index 00000000000..1c41ad8284e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java @@ -0,0 +1,694 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.expressiontransforms; + +import com.google.common.base.Joiner; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.searchdefinition.FeatureNames; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.integration.onnx.OnnxImporter; +import com.yahoo.searchlib.rankingexpression.integration.onnx.OnnxModel; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.Join; +import com.yahoo.tensor.functions.Reduce; +import com.yahoo.tensor.functions.Rename; +import com.yahoo.tensor.functions.ScalarFunctions; +import com.yahoo.tensor.functions.TensorFunction; +import com.yahoo.tensor.serialization.TypedBinaryFormat; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Replaces instances of the onnx(model-path, output) + * pseudofeature with the native Vespa ranking expression implementing + * the same computation. + * + * @author bratseth + * @author lesters + */ +public class OnnxFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + + private final OnnxImporter onnxImporter = new OnnxImporter(); + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map<Path, OnnxModel> importedModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("onnx")) return feature; + + try { + ModelStore store = new ModelStore(context.rankProfile().getSearch().sourceApplication(), feature.getArguments()); + if ( ! store.hasStoredModel()) // not converted yet - access Onnx model files + return transformFromOnnxModel(store, context.rankProfile(), context.queryProfiles()); + else + return transformFromStoredModel(store, context.rankProfile()); + } + catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use Onnx model from " + feature, e); + } + } + + private ExpressionNode transformFromOnnxModel(ModelStore store, + RankProfile profile, + QueryProfileRegistry queryProfiles) { + OnnxModel model = importedModels.computeIfAbsent(store.arguments().modelPath(), + k -> onnxImporter.importModel(store.arguments().modelName(), + store.onnxModelDir())); + + // Add constants + Set<String> constantsReplacedByMacros = new HashSet<>(); + model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); + model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles, + constantsReplacedByMacros, k, v)); + + // Find the specified expression + String output = chooseOutput(model, store.arguments().output()); + if (model.skippedOutputs().containsKey(output)) { + String message = "Could not import Onnx model output '" + output + "'"; + if (!model.skippedOutputs().get(output).isEmpty()) { + message += ": " + model.skippedOutputs().get(output); + } + if (!model.importWarnings().isEmpty()) { + message += ": " + String.join(", ", model.importWarnings()); + } + throw new IllegalArgumentException(message); + } + + RankingExpression expression = model.expressions().get(output); + expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + verifyRequiredMacros(expression, model, profile, queryProfiles); + addGeneratedMacros(model, profile); + reduceBatchDimensions(expression, model, profile, queryProfiles); + + model.macros().forEach((k, v) -> transformGeneratedMacro(store, profile, constantsReplacedByMacros, k, v)); + + store.writeConverted(expression); + return expression.getRoot(); + } + + private ExpressionNode transformFromStoredModel(ModelStore store, RankProfile profile) { + for (Pair<String, Tensor> constant : store.readSmallConstants()) + profile.addConstant(constant.getFirst(), asValue(constant.getSecond())); + + for (RankingConstant constant : store.readLargeConstants()) { + if ( ! profile.getSearch().getRankingConstants().containsKey(constant.getName())) + profile.getSearch().addRankingConstant(constant); + } + + for (Pair<String, RankingExpression> macro : store.readMacros()) { + addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond()); + } + + return store.readConverted().getRoot(); + } + + /** + * Returns the specified, existing output expression, or the only output expression if no output name is specified. + * Throws IllegalArgumentException in all other cases. + */ + private String chooseOutput(OnnxModel model, Optional<String> outputName) { + if ( ! outputName.isPresent()) { + if (model.outputs().size() == 0) + throw new IllegalArgumentException("No outputs are available" + skippedOutputsDescription(model)); + if (model.outputs().size() > 1) + throw new IllegalArgumentException("Onnx model has multiple outputs (" + + Joiner.on(", ").join(model.outputs().keySet()) + + "), one must be specified " + + "as a second argument to onnx()"); + return model.outputs().get(model.outputs().keySet().stream().findFirst().get()); + } + else { + String output = model.outputs().get(outputName.get()); + if (output == null) { + if (model.skippedOutputs().containsKey(outputName.get())) + throw new IllegalArgumentException("Could not use output '" + outputName.get() + "': " + + model.skippedOutputs().get(outputName.get())); + else + throw new IllegalArgumentException("Model does not have the specified output '" + + outputName.get() + "'"); + } + return output; + } + } + + private void transformSmallConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) { + store.writeSmallConstant(constantName, constantValue); + profile.addConstant(constantName, asValue(constantValue)); + } + + private void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, + Set<String> constantsReplacedByMacros, + String constantName, Tensor constantValue) { + RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); + if (macroOverridingConstant != null) { + TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); + if ( ! macroType.equals(constantValue.type())) + throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + + "The required type of this is " + constantValue.type() + + ", but the macro returns " + macroType); + constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later + } + else { + Path constantPath = store.writeLargeConstant(constantName, constantValue); + if ( ! profile.getSearch().getRankingConstants().containsKey(constantName)) { + profile.getSearch().addRankingConstant(new RankingConstant(constantName, constantValue.type(), + constantPath.toString())); + } + } + } + + private void transformGeneratedMacro(ModelStore store, RankProfile profile, + Set<String> constantsReplacedByMacros, + String macroName, RankingExpression expression) { + + expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + store.writeMacro(macroName, expression); + } + + private void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { + if (profile.getMacros().containsKey(macroName)) { + throw new IllegalArgumentException("Generated Onnx macro '" + macroName + "' already exists."); + } + profile.addMacro(macroName, false); // todo: inline if only used once + RankProfile.Macro macro = profile.getMacros().get(macroName); + macro.setRankingExpression(expression); + macro.setTextualExpression(expression.getRoot().toString()); + } + + private String skippedOutputsDescription(OnnxModel model) { + if (model.skippedOutputs().isEmpty()) return ""; + StringBuilder b = new StringBuilder(": "); + model.skippedOutputs().forEach((k, v) -> b.append("Skipping output '").append(k).append("': ").append(v)); + return b.toString(); + } + + /** + * Verify that the macros referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredMacros. + */ + private void verifyRequiredMacros(RankingExpression expression, OnnxModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + Set<String> macroNames = new HashSet<>(); + addMacroNamesIn(expression.getRoot(), macroNames, model); + for (String macroName : macroNames) { + TensorType requiredType = model.requiredMacros().get(macroName); + if (requiredType == null) continue; // Not a required macro + + RankProfile.Macro macro = profile.getMacros().get(macroName); + if (macro == null) + throw new IllegalArgumentException("Model refers Placeholder '" + macroName + + "' of type " + requiredType + " but this macro is not present in " + + profile); + // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second + // phase and summary features), as it may only resolve correctly given those bindings + // Or, probably better, annotate the macros with type constraints here and verify during general + // type verification + TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); + if ( actualType == null) + throw new IllegalArgumentException("Model refers input '" + macroName + + "' of type " + requiredType + + " which must be produced by a macro in the rank profile, but " + + "this macro references a feature which is not declared"); + if ( ! actualType.isAssignableTo(requiredType)) + throw new IllegalArgumentException("Model refers input '" + macroName + + "' of type " + requiredType + + " which must be produced by a macro in the rank profile, but " + + "this macro produces type " + actualType); + } + } + + /** + * Add the generated macros to the rank profile + */ + private void addGeneratedMacros(OnnxModel model, RankProfile profile) { + model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v)); + } + + /** + * Check if batch dimensions of inputs can be reduced out. If the input + * macro specifies that a single exemplar should be evaluated, we can + * reduce the batch dimension out. + */ + private void reduceBatchDimensions(RankingExpression expression, OnnxModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + TypeContext<Reference> typeContext = profile.typeContext(queryProfiles); + TensorType typeBeforeReducing = expression.getRoot().type(typeContext); + + // Check generated macros for inputs to reduce + Set<String> macroNames = new HashSet<>(); + addMacroNamesIn(expression.getRoot(), macroNames, model); + for (String macroName : macroNames) { + if ( ! model.macros().containsKey(macroName)) { + continue; + } + RankProfile.Macro macro = profile.getMacros().get(macroName); + if (macro == null) { + throw new IllegalArgumentException("Model refers to generated macro '" + macroName + + "but this macro is not present in " + profile); + } + RankingExpression macroExpression = macro.getRankingExpression(); + macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); + } + + // Check expression for inputs to reduce + ExpressionNode root = expression.getRoot(); + root = reduceBatchDimensionsAtInput(root, model, typeContext); + TensorType typeAfterReducing = root.type(typeContext); + root = expandBatchDimensionsAtOutput(root, typeBeforeReducing, typeAfterReducing); + expression.setRoot(root); + } + + private ExpressionNode reduceBatchDimensionsAtInput(ExpressionNode node, OnnxModel model, + TypeContext<Reference> typeContext) { + if (node instanceof TensorFunctionNode) { + TensorFunction tensorFunction = ((TensorFunctionNode) node).function(); + if (tensorFunction instanceof Rename) { + List<ExpressionNode> children = ((TensorFunctionNode)node).children(); + if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode) children.get(0); + if (model.requiredMacros().containsKey(referenceNode.getName())) { + return reduceBatchDimensionExpression(tensorFunction, typeContext); + } + } + } + } + if (node instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode) node; + if (model.requiredMacros().containsKey(referenceNode.getName())) { + return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); + } + } + if (node instanceof CompositeNode) { + List<ExpressionNode> children = ((CompositeNode)node).children(); + List<ExpressionNode> transformedChildren = new ArrayList<>(children.size()); + for (ExpressionNode child : children) { + transformedChildren.add(reduceBatchDimensionsAtInput(child, model, typeContext)); + } + return ((CompositeNode)node).setChildren(transformedChildren); + } + return node; + } + + private ExpressionNode reduceBatchDimensionExpression(TensorFunction function, TypeContext<Reference> context) { + TensorFunction result = function; + TensorType type = function.type(context); + if (type.dimensions().size() > 1) { + List<String> reduceDimensions = new ArrayList<>(); + for (TensorType.Dimension dimension : type.dimensions()) { + if (dimension.size().orElse(-1L) == 1) { + reduceDimensions.add(dimension.name()); + } + } + if (reduceDimensions.size() > 0) { + result = new Reduce(function, Reduce.Aggregator.sum, reduceDimensions); + } + } + return new TensorFunctionNode(result); + } + + /** + * If batch dimensions have been reduced away above, bring them back here + * for any following computation of the tensor. + * Todo: determine when this is not necessary! + */ + private ExpressionNode expandBatchDimensionsAtOutput(ExpressionNode node, TensorType before, TensorType after) { + if (after.equals(before)) { + return node; + } + TensorType.Builder typeBuilder = new TensorType.Builder(); + for (TensorType.Dimension dimension : before.dimensions()) { + if (dimension.size().orElse(-1L) == 1 && !after.dimensionNames().contains(dimension.name())) { + typeBuilder.indexed(dimension.name(), 1); + } + } + TensorType expandDimensionsType = typeBuilder.build(); + if (expandDimensionsType.dimensions().size() > 0) { + ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1.0)); + Generate generatedFunction = new Generate(expandDimensionsType, + new GeneratorLambdaFunctionNode(expandDimensionsType, + generatedExpression) + .asLongListToDoubleOperator()); + Join expand = new Join(TensorFunctionNode.wrapArgument(node), generatedFunction, ScalarFunctions.multiply()); + return new TensorFunctionNode(expand); + } + return node; + } + + /** + * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions. + * This method does that for the given expression and returns the result. + */ + private RankingExpression replaceConstantsByMacros(RankingExpression expression, + Set<String> constantsReplacedByMacros) { + if (constantsReplacedByMacros.isEmpty()) return expression; + return new RankingExpression(expression.getName(), + replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros)); + } + + private ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set<String> constantsReplacedByMacros) { + if (node instanceof ReferenceNode) { + Reference reference = ((ReferenceNode)node).reference(); + if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) { + String argument = reference.simpleArgument().get(); + if (constantsReplacedByMacros.contains(argument)) + return new ReferenceNode(argument); + } + } + if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above + CompositeNode composite = (CompositeNode)node; + return composite.setChildren(composite.children().stream() + .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros)) + .collect(Collectors.toList())); + } + return node; + } + + private void addMacroNamesIn(ExpressionNode node, Set<String> names, OnnxModel model) { + if (node instanceof ReferenceNode) { + ReferenceNode referenceNode = (ReferenceNode)node; + if (referenceNode.getOutput() == null) { // macro references cannot specify outputs + names.add(referenceNode.getName()); + if (model.macros().containsKey(referenceNode.getName())) { + addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model); + } + } + } + else if (node instanceof CompositeNode) { + for (ExpressionNode child : ((CompositeNode)node).children()) + addMacroNamesIn(child, names, model); + } + } + + private Value asValue(Tensor tensor) { + if (tensor.type().rank() == 0) + return new DoubleValue(tensor.asDouble()); // the backend gets offended by dimensionless tensors + else + return new TensorValue(tensor); + } + + /** + * Provides read/write access to the correct directories of the application package given by the feature arguments + */ + private static class ModelStore { + + private final ApplicationPackage application; + private final FeatureArguments arguments; + + public ModelStore(ApplicationPackage application, Arguments arguments) { + this.application = application; + this.arguments = new FeatureArguments(arguments); + } + + public FeatureArguments arguments() { return arguments; } + + public boolean hasStoredModel() { + try { + return application.getFile(arguments.expressionPath()).exists(); + } + catch (UnsupportedOperationException e) { + return false; + } + } + + /** + * Returns the directory which contains the source model to use for these arguments + */ + public File onnxModelDir() { + return application.getFileReference(ApplicationPackage.MODELS_DIR.append(arguments.modelPath())); + } + + /** + * Adds this expression to the application package, such that it can be read later. + */ + public void writeConverted(RankingExpression expression) { + application.getFile(arguments.expressionPath()) + .writeFile(new StringReader(expression.getRoot().toString())); + } + + /** Reads the previously stored ranking expression for these arguments */ + public RankingExpression readConverted() { + try { + return new RankingExpression(application.getFile(arguments.expressionPath()).createReader()); + } + catch (IOException e) { + throw new UncheckedIOException("Could not read " + arguments.expressionPath(), e); + } + catch (ParseException e) { + throw new IllegalStateException("Could not parse " + arguments.expressionPath(), e); + } + } + + /** Adds this macro expression to the application package to it can be read later. */ + public void writeMacro(String name, RankingExpression expression) { + application.getFile(arguments.macrosPath()).appendFile(name + "\t" + + expression.getRoot().toString() + "\n"); + } + + /** Reads the previously stored macro expressions for these arguments */ + public List<Pair<String, RankingExpression>> readMacros() { + try { + ApplicationFile file = application.getFile(arguments.macrosPath()); + if (!file.exists()) return Collections.emptyList(); + + List<Pair<String, RankingExpression>> macros = new ArrayList<>(); + BufferedReader reader = new BufferedReader(file.createReader()); + String line; + while (null != (line = reader.readLine())) { + String[] parts = line.split("\t"); + String name = parts[0]; + try { + RankingExpression expression = new RankingExpression(parts[1]); + macros.add(new Pair<>(name, expression)); + } + catch (ParseException e) { + throw new IllegalStateException("Could not parse " + arguments.expressionPath(), e); + } + } + return macros; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Reads the information about all the large (aka ranking) constants stored in the application package + * (the constant value itself is replicated with file distribution). + */ + public List<RankingConstant> readLargeConstants() { + try { + List<RankingConstant> constants = new ArrayList<>(); + for (ApplicationFile constantFile : application.getFile(arguments.largeConstantsPath()).listFiles()) { + String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); + constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); + } + return constants; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Adds this constant to the application package as a file, + * such that it can be distributed using file distribution. + * + * @return the path to the stored constant, relative to the application package root + */ + public Path writeLargeConstant(String name, Tensor constant) { + Path constantsPath = ApplicationPackage.MODELS_GENERATED_DIR.append(arguments.modelPath).append("constants"); + + // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: + Path constantPath = constantsPath.append(name + ".tbf"); + + // Remember the constant in a file we replicate in ZooKeeper + application.getFile(arguments.largeConstantsPath().append(name + ".constant")) + .writeFile(new StringReader(name + ":" + constant.type() + ":" + correct(constantPath))); + + // Write content explicitly as a file on the file system as this is distributed using file distribution + createIfNeeded(constantsPath); + IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant)); + return correct(constantPath); + } + + private List<Pair<String, Tensor>> readSmallConstants() { + try { + ApplicationFile file = application.getFile(arguments.smallConstantsPath()); + if (!file.exists()) return Collections.emptyList(); + + List<Pair<String, Tensor>> constants = new ArrayList<>(); + BufferedReader reader = new BufferedReader(file.createReader()); + String line; + while (null != (line = reader.readLine())) { + String[] parts = line.split("\t"); + String name = parts[0]; + TensorType type = TensorType.fromSpec(parts[1]); + Tensor tensor = Tensor.from(type, parts[2]); + constants.add(new Pair<>(name, tensor)); + } + return constants; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Append this constant to the single file used for small constants distributed as config + */ + public void writeSmallConstant(String name, Tensor constant) { + // Secret file format for remembering constants: + application.getFile(arguments.smallConstantsPath()).appendFile(name + "\t" + + constant.type().toString() + "\t" + + constant.toString() + "\n"); + } + + /** Workaround for being constructed with the .preprocessed dir as root while later being used outside it */ + private Path correct(Path path) { + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && ! path.elements().contains(FilesApplicationPackage.preprocessed)) { + return Path.fromString(FilesApplicationPackage.preprocessed).append(path); + } + else { + return path; + } + } + + private void createIfNeeded(Path path) { + File dir = application.getFileReference(path); + if ( ! dir.exists()) { + if (!dir.mkdirs()) + throw new IllegalStateException("Could not create " + dir); + } + } + + } + + /** Encapsulates the 1, 2 or 3 arguments to a onnx feature */ + private static class FeatureArguments { + + private final Path modelPath; + + /** Optional arguments */ + private final Optional<String> output; + + public FeatureArguments(Arguments arguments) { + if (arguments.isEmpty()) + throw new IllegalArgumentException("An onnx node must take an argument pointing to " + + "the onnx model directory under [application]/models"); + if (arguments.expressions().size() > 3) + throw new IllegalArgumentException("An onnx feature can have at most 2 arguments"); + + modelPath = Path.fromString(asString(arguments.expressions().get(0))); + output = optionalArgument(1, arguments); + } + + /** Returns modelPath with slashes replaced by underscores */ + public String modelName() { return modelPath.toString().replace('/', '_').replace('.', '_'); } + + /** Returns relative path to this model below the "models/" dir in the application package */ + public Path modelPath() { return modelPath; } + public Optional<String> output() { return output; } + + /** Path to the small constants file */ + public Path smallConstantsPath() { + return ApplicationPackage.MODELS_GENERATED_DIR.append(modelPath).append("constants.txt"); + } + + /** Path to the large (ranking) constants directory */ + public Path largeConstantsPath() { + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(modelPath).append("constants"); + } + + /** Path to the macros file */ + public Path macrosPath() { + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(modelPath).append("macros.txt"); + } + + public Path expressionPath() { + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR + .append(modelPath).append("expressions").append(expressionFileName()); + } + + private String expressionFileName() { + StringBuilder fileName = new StringBuilder(); + output.ifPresent(s -> fileName.append(s).append(".")); + if (fileName.length() == 0) // single signature and output + fileName.append("single."); + fileName.append("expression"); + return fileName.toString(); + } + + private Optional<String> optionalArgument(int argumentIndex, Arguments arguments) { + if (argumentIndex >= arguments.expressions().size()) + return Optional.empty(); + return Optional.of(asString(arguments.expressions().get(argumentIndex))); + } + + private String asString(ExpressionNode node) { + if ( ! (node instanceof ConstantNode)) + throw new IllegalArgumentException("Expected a constant string as onnx argument, but got '" + node); + return stripQuotes(((ConstantNode)node).sourceString()); + } + + private String stripQuotes(String s) { + if ( ! isQuoteSign(s.codePointAt(0))) return s; + if ( ! isQuoteSign(s.codePointAt(s.length() - 1 ))) + throw new IllegalArgumentException("onnx argument [" + s + "] is missing endquote"); + return s.substring(1, s.length()-1); + } + + private boolean isQuoteSign(int c) { + return c == '\'' || c == '"'; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index 5790a5294eb..41da32f64c3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -127,7 +127,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil addGeneratedMacros(model, profile); reduceBatchDimensions(expression, model, profile, queryProfiles); - model.macros().forEach((k, v) -> transformGeneratedMacro(store, profile, constantsReplacedByMacros, k, v)); + model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k, v)); store.writeConverted(expression); return expression.getRoot(); @@ -215,8 +215,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); if ( ! macroType.equals(constantValue.type())) throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + - "The required type of this is " + constantValue.type() + - ", but the macro returns " + macroType); + typeMismatchExplanation(constantValue.type(), macroType)); constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later } else { @@ -228,7 +227,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } } - private void transformGeneratedMacro(ModelStore store, RankProfile profile, + private void transformGeneratedMacro(ModelStore store, Set<String> constantsReplacedByMacros, String macroName, RankingExpression expression) { @@ -267,7 +266,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil RankProfile.Macro macro = profile.getMacros().get(macroName); if (macro == null) - throw new IllegalArgumentException("Model refers Placeholder '" + macroName + + throw new IllegalArgumentException("Model refers placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second @@ -276,18 +275,23 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil // type verification TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) - throw new IllegalArgumentException("Model refers Placeholder '" + macroName + + throw new IllegalArgumentException("Model refers placeholder '" + macroName + "' of type " + requiredType + " which must be produced by a macro in the rank profile, but " + "this macro references a feature which is not declared"); if ( ! actualType.isAssignableTo(requiredType)) - throw new IllegalArgumentException("Model refers Placeholder '" + macroName + - "' of type " + requiredType + - " which must be produced by a macro in the rank profile, but " + - "this macro produces type " + actualType); + throw new IllegalArgumentException("Model refers placeholder '" + macroName + "'. " + + typeMismatchExplanation(requiredType, actualType)); } } + private String typeMismatchExplanation(TensorType requiredType, TensorType actualType) { + return "The required type of this is " + requiredType + ", but this macro returns " + actualType + + (actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " + + "in query profile types - see the documentation." + : ""); + } + /** * Add the generated macros to the rank profile */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 868f092fc01..73fb532cfb4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -5,6 +5,7 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ValidationId; @@ -204,6 +205,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return fileDistributor; } + @Override + public Set<FileReference> fileReferences() { + return fileDistributor.allFilesToSend(); + } + /** Returns this models Vespa instance */ public ApplicationConfigProducerRoot getVespa() { return root; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 721e1c08989..7f8ff6edd85 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -48,8 +48,9 @@ public class RankSetupValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { + File cfgDir = null; try { - File cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile(); + cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile(); for (AbstractSearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet @@ -66,9 +67,12 @@ public class RankSetupValidator extends Validator { } } } - deleteTempDir(cfgDir); + } catch (IOException e) { throw new RuntimeException(e); + } finally { + if (cfgDir != null) + deleteTempDir(cfgDir); } } @@ -93,9 +97,7 @@ public class RankSetupValidator extends Validator { } private void deleteTempDir(File dir) { - if (!IOUtils.recursiveDeleteDir(dir)) { - throw new RuntimeException("Failed deleting " + dir); - } + IOUtils.recursiveDeleteDir(dir); } private void writeConfigs(String dir, AbstractConfigProducer producer) throws IOException { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 7444224258e..9801eab9f2b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -27,7 +27,6 @@ import java.util.TreeSet; /** * @author Einar M R Rosenvinge - * @since 5.1.11 */ public class ContainerDocumentApi implements FeederConfig.Producer { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index e60aabd24e8..659a07cfd5c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -60,7 +60,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl return s.toString(); } - private int getGCInterval(ModelElement documentNode) throws ParseException { + private int getGCInterval(ModelElement documentNode) { int gcInterval = 3600; if (documentNode != null) { gcInterval = documentNode.getIntegerAttribute("garbage-collection-interval", gcInterval); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java index 4bdef0607a2..0661ef67131 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -21,45 +21,11 @@ import static java.util.stream.Collectors.toSet; public class GlobalDistributionValidator { public void validate(Map<String, NewDocumentType> documentDefinitions, - Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy, - boolean enableMultipleBucketSpaces) { - if (!enableMultipleBucketSpaces) { - verifyGlobalDocumentsHaveRequiredRedundancy(globallyDistributedDocuments, redundancy); - verifySearchableCopiesIsSameAsRedundancy(globallyDistributedDocuments, redundancy); - } + Set<NewDocumentType> globallyDistributedDocuments) { verifyReferredDocumentsArePresent(documentDefinitions); verifyReferredDocumentsAreGlobal(documentDefinitions, globallyDistributedDocuments); } - private static void verifyGlobalDocumentsHaveRequiredRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && !redundancy.isEffectivelyGloballyDistributed()) { - throw new IllegalArgumentException( - String.format( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: %s. " + - "Redundancy is %d, expected %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveFinalRedundancy(), - redundancy.totalNodes())); - } - } - - private static void verifySearchableCopiesIsSameAsRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && - redundancy.effectiveReadyCopies() != redundancy.effectiveFinalRedundancy()) { - throw new IllegalArgumentException( - String.format( - "The following document types have the number of searchable copies less than redundancy: %s. " + - "Searchable copies is %d, while redundancy is %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveReadyCopies(), - redundancy.effectiveFinalRedundancy())); - } - } - private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) .filter(name -> !documentDefinitions.containsKey(name.toString())) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0119bced095..68ae4d2b242 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -61,14 +61,12 @@ public class ContentCluster extends AbstractConfigProducer implements MessagetyperouteselectorpolicyConfig.Producer, BucketspacesConfig.Producer { - // TODO: Make private private String documentSelection; private ContentSearchCluster search; private final boolean isHostedVespa; private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; - // Experimental flag (TODO: remove when feature is enabled by default) - private boolean enableMultipleBucketSpaces = false; + private boolean forceEnableMultipleBucketSpaces = false; private com.yahoo.vespa.model.content.StorageGroup rootGroup; private StorageCluster storageNodes; private DistributorCluster distributorNodes; @@ -250,7 +248,7 @@ public class ContentCluster extends AbstractConfigProducer implements private void setupExperimental(ContentCluster cluster, ModelElement experimental) { Boolean enableMultipleBucketSpaces = experimental.childAsBoolean("enable-multiple-bucket-spaces"); if (enableMultipleBucketSpaces != null) { - cluster.enableMultipleBucketSpaces = enableMultipleBucketSpaces; + cluster.forceEnableMultipleBucketSpaces = enableMultipleBucketSpaces; } } @@ -596,13 +594,13 @@ public class ContentCluster extends AbstractConfigProducer implements builder.min_distributor_up_ratio(0); builder.min_storage_up_ratio(0); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + builder.enable_multiple_bucket_spaces(true); // Telling the controller whether we actually _have_ global document types lets // it selectively enable or disable constraints that aren't needed when these // are not are present, even if full protocol and backend support is enabled // for multiple bucket spaces. Basically, if you don't use it, you don't // pay for it. - builder.cluster_has_global_document_types(enableMultipleBucketSpaces && !globallyDistributedDocuments.isEmpty()); + builder.cluster_has_global_document_types(!globallyDistributedDocuments.isEmpty()); } @Override @@ -646,7 +644,7 @@ public class ContentCluster extends AbstractConfigProducer implements } } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); - new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy, enableMultipleBucketSpaces); + new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); } public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>(); @@ -727,11 +725,13 @@ public class ContentCluster extends AbstractConfigProducer implements for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = ((enableMultipleBucketSpaces && isGloballyDistributed(docType)) - ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); docTypeBuilder.bucketspace(bucketSpace); builder.documenttype(docTypeBuilder); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + // NOTE: this config is kept around to allow the use of multiple bucket spaces + // on older versions of Vespa. It is for all intents and purposes a no-op in + // newer versions where multiple bucket spaces are enabled by default. + builder.enable_multiple_bucket_spaces(forceEnableMultipleBucketSpaces); } } diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..9e6b5cea55e --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -0,0 +1,40 @@ +attribute[].name "elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].name "elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype ARRAY +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..c1679c57d1a --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 252850086 +classes[].id 252850086 +classes[].name "default" +classes[].fields[].name "elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..8956a146b74 --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[].field "rankfeatures" +override[].command "rankfeatures" +override[].arguments "" +override[].field "summaryfeatures" +override[].command "summaryfeatures" +override[].arguments "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd new file mode 100644 index 00000000000..5b2d50cbdba --- /dev/null +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -0,0 +1,17 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_array type array<elem> { + indexing: summary + struct-field name { + indexing: attribute + } + struct-field weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..604fc1f6ea7 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -0,0 +1,60 @@ +attribute[0].name "elem_map.key" +attribute[0].datatype STRING +attribute[0].collectiontype ARRAY +attribute[0].removeifzero false +attribute[0].createifnonexistent false +attribute[0].fastsearch false +attribute[0].huge false +attribute[0].sortascending true +attribute[0].sortfunction UCA +attribute[0].sortstrength PRIMARY +attribute[0].sortlocale "" +attribute[0].enablebitvectors false +attribute[0].enableonlybitvector false +attribute[0].fastaccess false +attribute[0].arity 8 +attribute[0].lowerbound -9223372036854775808 +attribute[0].upperbound 9223372036854775807 +attribute[0].densepostinglistthreshold 0.4 +attribute[0].tensortype "" +attribute[0].imported false +attribute[1].name "elem_map.value.name" +attribute[1].datatype STRING +attribute[1].collectiontype ARRAY +attribute[1].removeifzero false +attribute[1].createifnonexistent false +attribute[1].fastsearch false +attribute[1].huge false +attribute[1].sortascending true +attribute[1].sortfunction UCA +attribute[1].sortstrength PRIMARY +attribute[1].sortlocale "" +attribute[1].enablebitvectors false +attribute[1].enableonlybitvector false +attribute[1].fastaccess false +attribute[1].arity 8 +attribute[1].lowerbound -9223372036854775808 +attribute[1].upperbound 9223372036854775807 +attribute[1].densepostinglistthreshold 0.4 +attribute[1].tensortype "" +attribute[1].imported false +attribute[2].name "elem_map.value.weight" +attribute[2].datatype INT32 +attribute[2].collectiontype ARRAY +attribute[2].removeifzero false +attribute[2].createifnonexistent false +attribute[2].fastsearch false +attribute[2].huge false +attribute[2].sortascending true +attribute[2].sortfunction UCA +attribute[2].sortstrength PRIMARY +attribute[2].sortlocale "" +attribute[2].enablebitvectors false +attribute[2].enableonlybitvector false +attribute[2].fastaccess false +attribute[2].arity 8 +attribute[2].lowerbound -9223372036854775808 +attribute[2].upperbound 9223372036854775807 +attribute[2].densepostinglistthreshold 0.4 +attribute[2].tensortype "" +attribute[2].imported false
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..7af49d95d09 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 653486243 +classes[0].id 653486243 +classes[0].name "default" +classes[0].fields[0].name "elem_map" +classes[0].fields[0].type "jsonstring" +classes[0].fields[1].name "rankfeatures" +classes[0].fields[1].type "featuredata" +classes[0].fields[2].name "summaryfeatures" +classes[0].fields[2].type "featuredata" +classes[0].fields[3].name "documentid" +classes[0].fields[3].type "longstring"
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..42b6e811ee6 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[0].field "rankfeatures" +override[0].command "rankfeatures" +override[0].arguments "" +override[1].field "summaryfeatures" +override[1].command "summaryfeatures" +override[1].arguments ""
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd new file mode 100644 index 00000000000..cb2eac4ed78 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -0,0 +1,20 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_map type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + } + struct-field value.name { + indexing: attribute + } + struct-field value.weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/integration/onnx/models/mnist_softmax.onnx b/config-model/src/test/integration/onnx/models/mnist_softmax.onnx Binary files differnew file mode 100644 index 00000000000..a86019bf53a --- /dev/null +++ b/config-model/src/test/integration/onnx/models/mnist_softmax.onnx diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index f113b10f508..d3e8136cdec 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchDefinition; -import org.json.JSONException; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -290,7 +289,7 @@ public class ApplicationDeployTest { } @Test - public void testConfigDefinitionsFromJars() throws IOException { + public void testConfigDefinitionsFromJars() { String appName = "src/test/cfg//application/app1"; FilesApplicationPackage app = FilesApplicationPackage.fromFile(new File(appName), false); Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs = app.getAllExistingConfigDefs(); @@ -298,31 +297,31 @@ public class ApplicationDeployTest { } @Test - public void testMetaData() throws IOException, JSONException { + public void testMetaData() throws IOException { File tmp = Files.createTempDir(); String appPkg = TESTDIR + "app1"; IOUtils.copyDirectory(new File(appPkg), tmp); - final DeployData deployData = new DeployData("foo", "bar", "baz", 13l, 1337l, 3l); + DeployData deployData = new DeployData("foo", "bar", "baz", 13l, false, 1337l, 3l); FilesApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); app.writeMetaData(); FilesApplicationPackage newApp = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); ApplicationMetaData meta = newApp.getMetaData(); assertThat(meta.getDeployedByUser(), is("foo")); assertThat(meta.getDeployPath(), is("bar")); - assertThat(meta.getDeployTimestamp(), is(13l)); - assertThat(meta.getGeneration(), is(1337l)); - assertThat(meta.getPreviousActiveGeneration(), is(3l)); - final String checkSum = meta.getCheckSum(); + assertThat(meta.getDeployTimestamp(), is(13L)); + assertThat(meta.getGeneration(), is(1337L)); + assertThat(meta.getPreviousActiveGeneration(), is(3L)); + String checkSum = meta.getCheckSum(); assertNotNull(checkSum); assertTrue((new File(tmp, "hosts.xml")).delete()); FilesApplicationPackage app2 = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); - final String app2CheckSum = app2.getMetaData().getCheckSum(); + String app2CheckSum = app2.getMetaData().getCheckSum(); assertThat(app2CheckSum, is(not(checkSum))); assertTrue((new File(tmp, "files/foo.json")).delete()); FilesApplicationPackage app3 = FilesApplicationPackage.fromFileWithDeployData(tmp, deployData); - final String app3CheckSum = app3.getMetaData().getCheckSum(); + String app3CheckSum = app3.getMetaData().getCheckSum(); assertThat(app3CheckSum, is(not(app2CheckSum))); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 95d5832b70d..4ee33abfc08 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -1,12 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.tensor.TensorType; import org.junit.Test; import java.io.IOException; +import java.util.Optional; import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -88,4 +92,48 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { assertTrue(attr.isFastAccess()); } + @Test + public void attribute_convert_to_array_copies_internal_state() { + StructDataType refType = new StructDataType("my_struct"); + Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, + Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType)); + single.setRemoveIfZero(true); + single.setCreateIfNonExistent(true); + single.setPrefetch(Boolean.TRUE); + single.setEnableBitVectors(true); + single.setEnableOnlyBitVector(true); + single.setFastSearch(true); + single.setHuge(true); + single.setFastAccess(true); + single.setPosition(true); + single.setArity(5); + single.setLowerBound(7); + single.setUpperBound(11); + single.setDensePostingListThreshold(13.3); + single.getSorting().setAscending(); + single.getAliases().add("foo"); + + Attribute array = single.convertToArray(); + assertEquals("foo", array.getName()); + assertEquals(Attribute.Type.STRING, array.getType()); + assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType()); + assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType()); + assertSame(single.referenceDocumentType(), array.referenceDocumentType()); + assertTrue(array.isRemoveIfZero()); + assertTrue(array.isCreateIfNonExistent()); + assertTrue(array.isPrefetch()); + assertTrue(array.isEnabledBitVectors()); + assertTrue(array.isEnabledOnlyBitVector()); + assertTrue(array.isFastSearch()); + assertTrue(array.isHuge()); + assertTrue(array.isFastAccess()); + assertTrue(array.isPosition()); + assertEquals(5, array.arity()); + assertEquals(7, array.lowerBound()); + assertEquals(11, array.upperBound()); + assertEquals(13.3, array.densePostingListThreshold(), 0.00001); + assertSame(single.getSorting(), array.getSorting()); + assertSame(single.getAliases(), array.getAliases()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 86f30ba3c11..c3cfcae66e6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Iterator; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -68,4 +69,47 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void array_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + + @Test + public void map_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { + assertEquals(name, attr.getName()); + assertEquals(type, attr.getType()); + assertEquals(collection, attr.getCollectionType()); + } + + @Test + public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { + Search search = SearchBuilder.createFromString( + joinLines("search test {", + " document test {", + " field pos_array type array<position> {", + " indexing: attribute", + " }", + " }", + "}")).getSearch(); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java index 9e73edf9b35..72c7aab4a39 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java @@ -18,4 +18,14 @@ public class AttributesTestCase extends AbstractExportingTestCase { assertCorrectDeriving("attributes"); } + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + + @Test + public void testMapOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_of_struct_attribute"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java new file mode 100644 index 00000000000..1c54d12d8b3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -0,0 +1,335 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.yolean.Exceptions; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +import com.yahoo.searchdefinition.processing.RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +public class RankingExpressionWithOnnxTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/onnx/"); + private final static String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_onnx_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_onnx_Variable_1), f(a,b)(a + b))"; + + @After + public void removeGeneratedConstantTensorFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testOnnxReference() throws ParseException { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx('mnist_softmax.onnx')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + @Test + public void testOnnxReferenceWithConstantFeature() { + RankProfileSearchFixture search = fixtureWith("constant(mytensor)", + "onnx('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor(d0[7],d1[784]) }", + null); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + @Test + public void testOnnxReferenceWithQueryFeature() { + String queryProfile = "<query-profile id='default' type='root'/>"; + String queryProfileType = "<query-profile-type id='root'>" + + " <field name='query(mytensor)' type='tensor(d0[3],d1[784])'/>" + + "</query-profile-type>"; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, + queryProfile, + queryProfileType); + RankProfileSearchFixture search = fixtureWith("query(mytensor)", + "onnx('mnist_softmax.onnx')", + null, + null, + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + @Test + public void testOnnxReferenceWithDocumentFeature() { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); + RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", + "onnx('mnist_softmax.onnx')", + null, + "field mytensor type tensor(d0[],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + + @Test + public void testOnnxReferenceWithFeatureCombination() { + String queryProfile = "<query-profile id='default' type='root'/>"; + String queryProfileType = "<query-profile-type id='root'>" + + " <field name='query(mytensor)' type='tensor(d0[3],d1[784],d2[10])'/>" + + "</query-profile-type>"; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, + queryProfile, + queryProfileType); + RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", + "onnx('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor(d0[7],d1[784]) }", + "field mytensor type tensor(d0[],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + + @Test + public void testNestedOnnxReference() { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "5 + sum(onnx('mnist_softmax.onnx'))"); + search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + } + + @Test + public void testOnnxReferenceSpecifyingOutput() { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx('mnist_softmax.onnx', 'add')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceMissingMacro() throws ParseException { + try { + RankProfileSearchFixture search = new RankProfileSearchFixture( + new StoringApplicationPackage(applicationDir), + new QueryProfileRegistry(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: onnx('mnist_softmax.onnx')" + + " }\n" + + " }"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx('mnist_softmax.onnx'): " + + "Model refers Placeholder 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "not present in rank profile 'my_profile'", + Exceptions.toMessageString(expected)); + } + } + + + @Test + public void testOnnxReferenceWithWrongMacroType() { + try { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", + "onnx('mnist_softmax.onnx')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx('mnist_softmax.onnx'): " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) which must be produced " + + "by a macro in the rank profile, but this macro produces type tensor(d0[2],d5[10])", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOnnxReferenceSpecifyingNonExistingOutput() { + try { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx('mnist_softmax.onnx', 'y')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx('mnist_softmax.onnx','y'): " + + "Model does not have the specified output 'y'", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx('mnist_softmax.onnx')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + assertLargeConstant("mnist_softmax_onnx_Variable", search, Optional.of(7840L)); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx('mnist_softmax.onnx')", + null, + null, + "Placeholder", + storedApplication); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + // Verify that the constants exists, but don't verify the content as we are not + // simulating file distribution in this test + assertLargeConstant("mnist_softmax_onnx_Variable_1", searchFromStored, Optional.empty()); + assertLargeConstant("mnist_softmax_onnx_Variable", searchFromStored, Optional.empty()); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + @Test + public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException { + String rankProfile = + " rank-profile my_profile {\n" + + " macro Placeholder() {\n" + + " expression: tensor(d0[2],d1[784])(0.0)\n" + + " }\n" + + " macro mnist_softmax_onnx_Variable() {\n" + + " expression: tensor(d1[10],d2[784])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx('mnist_softmax.onnx')" + + " }\n" + + " }"; + + + String vespaExpressionWithoutConstant = + "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), mnist_softmax_onnx_Variable, f(a,b)(a * b)), sum, d2), constant(mnist_softmax_onnx_Variable_1), f(a,b)(a + b))"; + RankProfileSearchFixture search = fixtureWith(rankProfile, new StoringApplicationPackage(applicationDir)); + search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + + assertNull("Constant overridden by macro is not added", + search.search().getRankingConstants().get("mnist_softmax_onnx_Variable")); + assertLargeConstant("mnist_softmax_onnx_Variable_1", search, Optional.of(10L)); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith(rankProfile, storedApplication); + searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + assertNull("Constant overridden by macro is not added", + searchFromStored.search().getRankingConstants().get("mnist_softmax_onnx_Variable")); + assertLargeConstant("mnist_softmax_onnx_Variable_1", searchFromStored, Optional.of(10L)); + } finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + /** + * Verifies that the constant with the given name exists, and - only if an expected size is given - + * that the content of the constant is available and has the expected size. + */ + private void assertLargeConstant(String name, RankProfileSearchFixture search, Optional<Long> expectedSize) { + try { + Path constantApplicationPackagePath = Path.fromString("models.generated/mnist_softmax.onnx/constants").append(name + ".tbf"); + RankingConstant rankingConstant = search.search().getRankingConstants().get(name); + assertEquals(name, rankingConstant.getName()); + assertTrue(rankingConstant.getFileName().endsWith(constantApplicationPackagePath.toString())); + + if (expectedSize.isPresent()) { + Path constantPath = applicationDir.append(constantApplicationPackagePath); + assertTrue("Constant file '" + constantPath + "' has been written", + constantPath.toFile().exists()); + Tensor deserializedConstant = TypedBinaryFormat.decode(Optional.empty(), + GrowableByteBuffer.wrap(IOUtils.readFileBytes(constantPath.toFile()))); + assertEquals(expectedSize.get().longValue(), deserializedConstant.size()); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) { + return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression, + String constant, String field) { + return fixtureWith(placeholderExpression, firstPhaseExpression, constant, field, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String rankProfile, StoringApplicationPackage application) { + try { + return new RankProfileSearchFixture(application, application.getQueryProfiles(), + rankProfile, null, null); + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + private RankProfileSearchFixture fixtureWith(String macroExpression, + String firstPhaseExpression, + String constant, + String field, + String macroName, + StoringApplicationPackage application) { + try { + return new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " macro " + macroName + "() {\n" + + " expression: " + macroExpression + + " }\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 623f26a6b27..d288a396732 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -162,7 +162,7 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers Placeholder 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers placeholder 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } @@ -179,8 +179,8 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers Placeholder 'Placeholder' of type tensor(d0[],d1[784]) which must be produced " + - "by a macro in the rank profile, but this macro produces type tensor(d0[2],d5[10])", + "Model refers placeholder 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + + "but this macro returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -439,7 +439,7 @@ public class RankingExpressionWithTensorFlowTestCase { } } - private static class StoringApplicationPackage extends MockApplicationPackage { + static class StoringApplicationPackage extends MockApplicationPackage { private final File root; @@ -465,7 +465,7 @@ public class RankingExpressionWithTensorFlowTestCase { } - private static class StoringApplicationPackageFile extends ApplicationFile { + public static class StoringApplicationPackageFile extends ApplicationFile { /** The path to the application package root */ private final Path root; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java new file mode 100644 index 00000000000..bc812e12f43 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java @@ -0,0 +1,63 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class ImplicitIndexingClusterTest { + @Test + public void existing_jdisc_is_used_as_indexing_cluster_when_multitenant() { + final String servicesXml = "<services version=\"1.0\">\n" + // + " <jdisc version=\"1.0\" id=\"jdisc\">\n" + // + " <search />\n" + // + " <nodes count=\"1\" />\n" + // + ACCESS_CONTROL_XML + // + " </jdisc>\n" + // + " <content id=\"music\" version=\"1.0\">\n" + // + " <redundancy>1</redundancy>\n" + // + " <documents>\n" + // + " <document type=\"music\" mode=\"index\" />\n" + // + " </documents>\n" + // + " <nodes count=\"1\" />\n" + // + " </content>\n" + // + "</services>\n"; + + + VespaModel vespaModel = buildMultiTenantVespaModel(servicesXml); + ContainerCluster jdisc = vespaModel.getContainerClusters().get("jdisc"); + assertNotNull("Docproc not added to jdisc", jdisc.getDocproc()); + assertNotNull("Indexing chain not added to jdisc", jdisc.getDocprocChains().allChains().getComponent("indexing")); + } + + private final String ACCESS_CONTROL_XML = "<http>\n" +// + " <filtering>\n" +// + " <access-control domain=\"foo\" />\n" +// + " </filtering>\n" +// + " <server id=\"bar\" port=\"4080\" />\n" +// + "</http>\n"; + + private static VespaModel buildMultiTenantVespaModel(String servicesXml) { + DeployProperties properties = new DeployProperties.Builder().multitenant(true).hostedVespa(true).build(); + DeployState.Builder deployStateBuilder = new DeployState.Builder() + .properties(properties) + .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com")); + + return new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + servicesXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java new file mode 100644 index 00000000000..bc6ff9e9be4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.java @@ -0,0 +1,180 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.container.search.searchchain.Source.GroupOption; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class FederationSearcherTest { + + private static class FederationFixture { + FederationSearcher federationSearchWithDefaultSources = newFederationSearcher(true, emptyList()); + private ComponentRegistry<SearchChain> searchChainRegistry = new ComponentRegistry<>(); + private SourceGroupRegistry sourceGroupRegistry = new SourceGroupRegistry(); + + void initializeFederationSearcher(FederationSearcher searcher) { + searcher.initialize(searchChainRegistry, sourceGroupRegistry); + } + + void registerProviderWithSources(Provider provider) { + List<GenericTarget> sources = new ArrayList<>(); + sources.add(provider); + sources.addAll(provider.getSources()); + for (GenericTarget gt : sources) { + searchChainRegistry.register(gt.getId(), gt); + } + sourceGroupRegistry.addSources(provider); + } + } + + private static class ProvidersWithSourceFixture extends FederationFixture { + Provider provider1 = createProvider(ComponentId.fromString("provider1")); + Provider provider2 = createProvider(ComponentId.fromString("provider2")); + + private ProvidersWithSourceFixture() { + super(); + provider1.addSource(createSource(ComponentId.fromString("source"), GroupOption.leader)); + provider2.addSource(createSource(ComponentId.fromString("source"), GroupOption.participant)); + + registerProviderWithSources(provider1); + registerProviderWithSources(provider2); + initializeFederationSearcher(federationSearchWithDefaultSources); + } + } + + @Test + public void default_providers_are_inherited_when_inheritDefaultSources_is_true() throws Exception { + FederationFixture f = new FederationFixture(); + + final String providerId = "providerId"; + + f.registerProviderWithSources(createProvider(ComponentId.fromString(providerId))); + f.initializeFederationSearcher(f.federationSearchWithDefaultSources); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + FederationConfig.Target target = federationConfig.target(0); + + assertSame(providerId, target.id()); // by identity + assertTrue("Not used by default", target.searchChain(0).useByDefault()); + } + + @Test + public void source_groups_are_inherited_when_inheritDefaultSources_is_true() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + Assert.assertEquals(1, federationConfig.target().size()); + + FederationConfig.Target target = federationConfig.target(0); + assertEquals(target.id(), "source"); + assertTrue("Not used by default", target.useByDefault()); + assertEquals(2, target.searchChain().size()); + assertThat(target.searchChain().stream().map(FederationConfig.Target.SearchChain::providerId).collect(toList()), + contains("provider1", "provider2")); + } + + @Test + public void source_groups_are_not_inherited_when_inheritDefaultSources_is_false() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationSearcher federationSearcherWithoutDefaultSources = newFederationSearcher(false, emptyList()); + f.initializeFederationSearcher(federationSearcherWithoutDefaultSources); + + FederationConfig federationConfig = getConfig(federationSearcherWithoutDefaultSources); + assertEquals(0, federationConfig.target().size()); + } + + @Test + public void leaders_must_be_the_first_search_chain_in_a_target() throws Exception { + FederationFixture f = new ProvidersWithSourceFixture(); + + FederationConfig federationConfig = getConfig(f.federationSearchWithDefaultSources); + List<FederationConfig.Target.SearchChain> searchChain = federationConfig.target(0).searchChain(); + + assertEquals("provider1", searchChain.get(0).providerId()); + assertEquals("provider2", searchChain.get(1).providerId()); + } + + @Test + public void manually_specified_targets_overrides_inherited_targets() throws Exception { + FederationFixture f = new FederationFixture(); + + f.registerProviderWithSources(createProvider(ComponentId.fromString("provider1"))); + FederationSearcher federation = newFederationSearcher(true, + singletonList(new TargetSpec(ComponentSpecification.fromString("provider1"), + new FederationOptions().setTimeoutInMilliseconds(12345)))); + f.initializeFederationSearcher(federation); + + FederationConfig federationConfig = getConfig(federation); + assertEquals(1, federationConfig.target().size()); + + FederationConfig.Target target = federationConfig.target(0); + assertEquals(1, target.searchChain().size()); + + FederationConfig.Target.SearchChain searchChain = target.searchChain(0); + assertEquals(12345, searchChain.timeoutMillis()); + } + + private static FederationSearcher newFederationSearcher(boolean inheritDefaultSources, List<TargetSpec> targets) { + return new FederationSearcher(new FederationSearcherModel(ComponentSpecification.fromString("federation"), + Dependencies.emptyDependencies(), targets, inheritDefaultSources), Optional.empty()); + } + + private static ChainSpecification searchChainSpecification(ComponentId id) { + return new ChainSpecification(id, new ChainSpecification.Inheritance(null, null), emptyList(), emptySet()); + } + + private static Provider createProvider(ComponentId id) { + return new Provider(searchChainSpecification(id), new FederationOptions()); + } + + private static Source createSource(ComponentId id, GroupOption groupOption) { + return new Source(searchChainSpecification(id), new FederationOptions(), groupOption); + } + + private static FederationConfig getConfig(ConfigProducer configProducer) throws Exception { + Optional<Class<?>> builderClassOpt = Arrays.stream(FederationConfig.class.getDeclaredClasses()) + .filter(c -> c.getSimpleName().equals("Builder")).findFirst(); + if (builderClassOpt.isPresent() == false) { + throw new RuntimeException("No Builder class in ConfigInstance."); + } + Class<?> builderClass = builderClassOpt.get(); + + Object builder = builderClass.getDeclaredConstructor().newInstance(); + Method getConfigMethod = configProducer.getClass().getMethod("getConfig", builderClass); + + getConfigMethod.invoke(configProducer, builder); + + return FederationConfig.class.getConstructor(builderClass).newInstance(builder); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java new file mode 100644 index 00000000000..c58f3308ced --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.search.grouping.GroupingValidator; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @author ollivir + */ + +public class BundleInstantiationSpecificationBuilderTest { + + @Test + public void bundle_is_not_replaced_for_user_defined_class() throws IOException, SAXException { + final String userDefinedClass = "my own class that will also be set as bundle"; + verifyExpectedBundle(userDefinedClass, null, userDefinedClass); + } + + @Test + public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { + String internalClass = GroupingValidator.class.getName(); + verifyExpectedBundle(internalClass, null, BundleMapper.searchAndDocprocBundle); + } + + @Test + public void bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() + throws IOException, SAXException { + String internalClass = GroupingValidator.class.getName(); + String explicitBundle = "my-own-implementation"; + verifyExpectedBundle(internalClass, explicitBundle, explicitBundle); + } + + private static void verifyExpectedBundle(String className, String explicitBundle, String expectedBundle) + throws IOException, SAXException { + String xml = "<component id=\"_\" class=\"" + className + "\""; + if (explicitBundle != null) { + xml += " bundle=\"" + explicitBundle + "\""; + } + xml += " />"; + InputStream xmlStream = IOUtils.toInputStream(xml); + Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); + + BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component, false); + assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 98177b4ada0..0156128f7ca 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -173,17 +173,17 @@ public class ContentSearchClusterTest { } @Test - public void require_that_all_document_types_belong_to_default_bucket_space_by_default() throws Exception { + public void require_that_document_types_belong_to_correct_bucket_spaces() throws Exception { BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType()); assertEquals(2, config.documenttype().size()); - assertDocumentType("global", "default", config.documenttype(0)); + assertDocumentType("global", "global", config.documenttype(0)); assertDocumentType("regular", "default", config.documenttype(1)); // Safeguard against flipping the switch assertFalse(config.enable_multiple_bucket_spaces()); } @Test - public void require_that_multiple_bucket_spaces_can_be_enabled() throws Exception { + public void require_that_multiple_bucket_spaces_can_be_force_enabled() throws Exception { ContentCluster cluster = createClusterWithMultipleBucketSpacesEnabled(); { BucketspacesConfig config = getBucketspacesConfig(cluster); @@ -210,9 +210,9 @@ public class ContentSearchClusterTest { } @Test - public void controller_global_documents_config_forced_to_false_if_multiple_spaces_not_enabled() throws Exception { + public void controller_global_documents_config_always_enabled_even_without_experimental_flag_set() throws Exception { ContentCluster cluster = createClusterWithGlobalDocsButNotMultipleSpacesEnabled(); - assertFalse(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); + assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java index b8252f2f081..6506f7a08a8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -26,64 +26,16 @@ public class GlobalDistributionValidatorTest { public final ExpectedException exceptionRule = ExpectedException.none(); @Test - public void throws_exception_if_redudancy_does_not_imply_global_distribution() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: " + - "'bar', 'foo'. Redundancy is 2, expected 3."); - validate(fixture, redundancy); - } - - @Test - public void validation_of_redundancy_is_deactivated_if_multiple_bucket_spaces_is_enabled() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies(); - - validate(fixture, redundancy, true); - } - - @Test - public void throws_exception_if_searchable_copies_too_low() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithTooFewSearchableCopies(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types have the number of searchable copies less than redundancy: " + - "'bar', 'foo'. Searchable copies is 1, while redundancy is 2."); - validate(fixture, redundancy); - } - - @Test - public void validation_succeeds_when_globally_distributed_and_enough_searchable_copies() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); - } - - @Test public void validation_succeeds_on_no_documents() { new GlobalDistributionValidator() - .validate(emptyMap(), emptySet(), createRedundancyWithoutGlobalDistribution(), false); + .validate(emptyMap(), emptySet()); } @Test public void validation_succeeds_on_no_global_documents() { Fixture fixture = new Fixture() .addNonGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -92,11 +44,10 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addNonGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -105,8 +56,7 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -115,11 +65,10 @@ public class GlobalDistributionValidatorTest { NewDocumentType child = createDocumentType("child", unknown); Fixture fixture = new Fixture() .addNonGlobalDocument(child); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -130,42 +79,14 @@ public class GlobalDistributionValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create(); } - private static Redundancy createRedundancyWithGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(3); - return redundancy; - } - - private static Redundancy createRedundancyWithTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(3); - return redundancy; - } - private static NewDocumentType createDocumentType(String name, NewDocumentType... references) { Set<NewDocumentType.Name> documentReferences = Stream.of(references).map(NewDocumentType::getFullName).collect(toSet()); return new NewDocumentType(new NewDocumentType.Name(name), documentReferences); } - private static void validate(Fixture fixture, Redundancy redundancy) { - validate(fixture, redundancy, false); - } - - private static void validate(Fixture fixture, Redundancy redundancy, boolean enableMultipleBucketSpaces) { + private static void validate(Fixture fixture) { new GlobalDistributionValidator() - .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments(), redundancy, enableMultipleBucketSpaces); + .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments()); } private static class Fixture { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java index eabd0e5a7e0..131a5344116 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java @@ -7,10 +7,10 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.test.MockHosts; import org.junit.Test; +import java.io.File; import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,5 +48,10 @@ public class FileDistributorTestCase { public void startDownload(String hostName, int port, Set<FileReference> fileReferences) { filesToDownloadCalled++; } + + @Override + public File getFileReferencesDir() { + return null; + } } } diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala deleted file mode 100644 index 4ebe14c1e85..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.scala +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search - -import com.yahoo.config.model.provision.InMemoryProvisioner -import com.yahoo.config.model.test.MockApplicationPackage -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg -import org.junit.Test - -import org.junit.Assert.assertNotNull -import scala.xml.{XML, Elem} -import java.io.StringWriter -import com.yahoo.config.model.deploy.{DeployProperties, DeployState} - -/** - * @author tonytv - */ -class ImplicitIndexingClusterTest { - @Test - def existing_jdisc_is_used_as_indexing_cluster_when_multitenant() { - val servicesXml = - <services version="1.0"> - <jdisc version="1.0" id="jdisc"> - <search /> - <nodes count="1" /> - {accessControlXml} - </jdisc> - <content id="music" version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index" /> - </documents> - <nodes count="1" /> - </content> - </services> - - - val vespaModel = buildMultiTenantVespaModel(servicesXml) - val jdisc = vespaModel.getContainerClusters.get("jdisc") - assertNotNull("Docproc not added to jdisc", jdisc.getDocproc) - assertNotNull("Indexing chain not added to jdisc", jdisc.getDocprocChains.allChains().getComponent("indexing")) - } - - private val accessControlXml = - <http> - <filtering> - <access-control domain="foo" /> - </filtering> - <server id="bar" port="4080" /> - </http> - - - def buildMultiTenantVespaModel(servicesXml: Elem) = { - val properties = new DeployProperties.Builder().multitenant(true).hostedVespa(true).build() - val deployStateBuilder = new DeployState.Builder() - .properties(properties) - .modelHostProvisioner(new InMemoryProvisioner(true, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com")) - - val writer = new StringWriter - XML.write(writer, servicesXml, "UTF-8", xmlDecl = true, doctype = null) - writer.close() - - new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(writer.toString) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder) - } -} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala deleted file mode 100644 index a1eeb297367..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/search/searchchain/FederationSearcherTest.scala +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search.searchchain - -import java.util.Collections.{emptyList, emptySet} -import java.util.Optional - -import com.yahoo.component.chain.dependencies.Dependencies -import com.yahoo.component.chain.model.ChainSpecification -import com.yahoo.component.provider.ComponentRegistry -import com.yahoo.component.{ComponentId, ComponentSpecification} -import com.yahoo.config.ConfigInstance -import com.yahoo.search.federation.FederationConfig -import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec -import com.yahoo.search.searchchain.model.federation.{FederationOptions, FederationSearcherModel} -import com.yahoo.vespa.model.ConfigProducer -import com.yahoo.vespa.model.container.search.searchchain.FederationSearcherTest._ -import com.yahoo.vespa.model.container.search.searchchain.Source.GroupOption -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner - -import scala.collection.JavaConverters._ -import scala.collection.breakOut -import scala.language.implicitConversions -import scala.reflect.ClassTag - -/** - * @author tonytv - */ -@RunWith(classOf[JUnitRunner]) -class FederationSearcherTest extends FunSuite{ - - class FederationFixture { - val federationSearchWithDefaultSources = newFederationSearcher(inheritDefaultSources = true) - val searchChainRegistry = new ComponentRegistry[SearchChain] - val sourceGroupRegistry = new SourceGroupRegistry - - def initializeFederationSearcher(searcher: FederationSearcher = federationSearchWithDefaultSources) { - searcher.initialize(searchChainRegistry, sourceGroupRegistry) - } - - def registerProviderWithSources(provider: Provider) = { - provider :: provider.getSources.asScala.toList foreach { chain => searchChainRegistry.register(chain.getId, chain) } - sourceGroupRegistry.addSources(provider) - } - } - - class ProvidersWithSourceFixture extends FederationFixture { - val provider1 = createProvider("provider1") - val provider2 = createProvider("provider2") - - provider1.addSource(createSource("source", GroupOption.leader)) - provider2.addSource(createSource("source", GroupOption.participant)) - - registerProviderWithSources(provider1) - registerProviderWithSources(provider2) - initializeFederationSearcher() - } - - test("default providers are inherited when inheritDefaultSources=true") { - val f = new FederationFixture - import f._ - - val providerId = "providerId" - - registerProviderWithSources(createProvider(providerId)) - initializeFederationSearcher() - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - val target = federationConfig.target(0) - - assert( providerId === target.id() ) - assert( target.searchChain(0).useByDefault(), "Not used by default" ) - } - - def toMapByKey[KEY, VALUE](collection: java.util.Collection[VALUE])(f: VALUE => KEY): Map[KEY, VALUE] = - collection.asScala.map(e => (f(e), e))(breakOut) - - test("source groups are inherited when inheritDefaultSources=true") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - assert(federationConfig.target().size == 1) - - val target = federationConfig.target(0) - assert(target.id() == "source") - assert(target.useByDefault(), "Not used by default") - - //val chainsByProviderId = toMapByKey(target.searchChain())(_.providerId()) - - assert(Set("provider1", "provider2") === target.searchChain().asScala.map(_.providerId()).toSet) - } - - test("source groups are not inherited when inheritDefaultSources=false") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationSearcherWithoutDefaultSources = newFederationSearcher(inheritDefaultSources = false) - initializeFederationSearcher(federationSearcherWithoutDefaultSources) - - val federationConfig = getConfig[FederationConfig](federationSearcherWithoutDefaultSources) - assert(federationConfig.target().size == 0) - } - - test("leaders must be the first search chain in a target") { - val f = new ProvidersWithSourceFixture - import f._ - - val federationConfig = getConfig[FederationConfig](federationSearchWithDefaultSources) - val searchChain = federationConfig.target(0).searchChain - - assert(searchChain.get(0).providerId() === "provider1") - assert(searchChain.get(1).providerId() === "provider2") - - } - - test("manually specified targets overrides inherited targets") { - val f = new FederationFixture - import f._ - - registerProviderWithSources(createProvider("provider1")) - val federation = newFederationSearcher(inheritDefaultSources = true, - targets = List(new TargetSpec("provider1", new FederationOptions().setTimeoutInMilliseconds(12345))).asJava) - - initializeFederationSearcher(federation) - - val federationConfig = getConfig[FederationConfig](federation) - - assert(federationConfig.target().size === 1) - val target = federationConfig.target(0) - - assert(target.searchChain().size === 1) - val searchChain = target.searchChain(0) - - assert(searchChain.timeoutMillis() === 12345) - } - - - def newFederationSearcher(inheritDefaultSources: Boolean, - targets: java.util.List[TargetSpec] = emptyList()): FederationSearcher = { - new FederationSearcher( - new FederationSearcherModel("federation", - Dependencies.emptyDependencies(), - targets, - inheritDefaultSources), - Optional.empty()) - } -} - -object FederationSearcherTest { - implicit def toComponentId(name: String): ComponentId = ComponentId.fromString(name) - implicit def toComponentSpecification(name: String): ComponentSpecification = ComponentSpecification.fromString(name) - - def newBuilder[T <: ConfigInstance.Builder](implicit c: ClassTag[T]): T = { - c.runtimeClass.getDeclaredConstructor().newInstance().asInstanceOf[T] - } - - def searchChainSpecification(id: ComponentId) = - new ChainSpecification(id, new ChainSpecification.Inheritance(null, null), emptyList(), emptySet()) - - def createProvider(id: ComponentId) = - new Provider(searchChainSpecification(id), new FederationOptions()) - - def createSource(id: ComponentId, groupOption: GroupOption) = - new Source(searchChainSpecification(id), new FederationOptions(), groupOption) - - - //TODO: TVT: move - def getConfig[T <: ConfigInstance : ClassTag](configProducer: ConfigProducer): T = { - val configClass = implicitly[ClassTag[T]].runtimeClass - val builderClass = configClass.getDeclaredClasses.collectFirst {case c if c.getSimpleName == "Builder" => c } getOrElse { - sys.error("No Builder class in ConfigInstance.") - } - - val builder = builderClass.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef] - val getConfigMethod = configProducer.getClass.getMethod("getConfig", builderClass) - - getConfigMethod.invoke(configProducer, builder) - - configClass.getConstructor(builderClass).newInstance(builder).asInstanceOf[T] - } -} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala deleted file mode 100644 index 1f849f91c2e..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.scala +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml - -import com.yahoo.component.ComponentSpecification -import com.yahoo.search.grouping.GroupingValidator - -import scala.language.implicitConversions -import BundleInstantiationSpecificationBuilderTest._ -import com.yahoo.config.model.builder.xml.test.DomBuilderTest -import org.hamcrest.CoreMatchers._ -import org.junit.Assert._ -import org.junit.Test -import org.w3c.dom.Element - -import scala.xml.Elem - -/** - * @author gjoranv - * @since 5.45 - */ - -class BundleInstantiationSpecificationBuilderTest { - - @Test - def bundle_is_not_replaced_for_user_defined_class() { - val userDefinedClass = "my own class that will also be set as bundle" - verifyExpectedBundle(userDefinedClass, - expectedBundle = userDefinedClass) - } - - @Test - def bundle_is_replaced_for_internal_class() = { - val internalClass = classOf[GroupingValidator].getName - verifyExpectedBundle(internalClass, - expectedBundle = BundleMapper.searchAndDocprocBundle) - } - - @Test - def bundle_is_not_replaced_for_internal_class_with_explicitly_set_bundle() = { - val internalClass = classOf[GroupingValidator].getName - val explicitBundle = "my-own-implementation" - verifyExpectedBundle(internalClass, - explicitBundle = Some(explicitBundle), - expectedBundle = explicitBundle) - } -} - -object BundleInstantiationSpecificationBuilderTest { - - def verifyExpectedBundle(className: String, - explicitBundle: Option[String] = None, - expectedBundle:String) = { - val xml = <component id="_" class={className} bundle={explicitBundle.orNull} /> - - val spec = BundleInstantiationSpecificationBuilder.build(xml, false) - assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))) - } - - implicit def toDomElement(elem: Elem): Element = { - DomBuilderTest.parse(elem.toString()) - } -} diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 68985bd598a..f80c2699da2 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -314,7 +314,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void returnOkResponse(JRTServerConfigRequest request, RawConfig config) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy returnOkResponse()"); - request.addOkResponse(config.getPayload(), config.getGeneration(), config.getConfigMd5()); + request.addOkResponse(config.getPayload(), config.getGeneration(), config.isInternalRedeploy(), config.getConfigMd5()); log.log(LogLevel.DEBUG, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + ",generation=" + config.getGeneration()); log.log(LogLevel.SPAM, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index cc88df0af37..8d88281207c 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN environment bootstrap section diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java index c16d6ccbcf5..33e798da15e 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java @@ -44,11 +44,11 @@ public class ConfigTester { String defMd5 = ConfigUtils.getDefMd5(defContent); String configMd5 = ConfigUtils.getMd5(fooConfigPayload); fooConfig = new RawConfig(configKey, defMd5, fooPayload, configMd5, - generation, defContent, Optional.empty()); + generation, false, defContent, Optional.empty()); String defName2 = "bar"; barConfig = new RawConfig(new ConfigKey<>(defName2, configId, namespace), defMd5, fooPayload, configMd5, - generation, defContent, Optional.empty()); + generation, false, defContent, Optional.empty()); } JRTServerConfigRequest createRequest(RawConfig config) { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java index 2f2aac2a6bb..16bc2c66a94 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java @@ -63,9 +63,9 @@ public class MemoryCacheTest { slime.setString("bar \"value2\""); payloadDifferentMd5 = Payload.from(new ConfigPayload(slime)); - config = new RawConfig(configKey, defMd5, payload, configMd5, generation, defContent, Optional.empty()); - config2 = new RawConfig(configKey2, defMd52, payload2, configMd5, generation, defContent, Optional.empty()); - configDifferentMd5 = new RawConfig(configKey, differentDefMd5, payloadDifferentMd5, configMd5, generation, defContent, Optional.empty()); + config = new RawConfig(configKey, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); + config2 = new RawConfig(configKey2, defMd52, payload2, configMd5, generation, false, defContent, Optional.empty()); + configDifferentMd5 = new RawConfig(configKey, differentDefMd5, payloadDifferentMd5, configMd5, generation, false, defContent, Optional.empty()); cacheKey = new ConfigCacheKey(configKey, config.getDefMd5()); cacheKey2 = new ConfigCacheKey(configKey2, config2.getDefMd5()); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index 3cd0f1043cc..513a5caa08d 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.*; /** * @author hmusum - * @since 5.1.9 */ public class ProxyServerTest { @@ -35,7 +34,7 @@ public class ProxyServerTest { private static final ConfigKey<?> errorConfigKey = new ConfigKey<>("error", fooConfig.getConfigId(), fooConfig.getNamespace()); static final RawConfig errorConfig = new RawConfig(errorConfigKey, fooConfig.getDefMd5(), fooConfig.getPayload(), fooConfig.getConfigMd5(), - fooConfig.getGeneration(), ErrorCode.UNKNOWN_DEFINITION, fooConfig.getDefContent(), Optional.empty()); + fooConfig.getGeneration(), false, ErrorCode.UNKNOWN_DEFINITION, fooConfig.getDefContent(), Optional.empty()); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -223,7 +222,8 @@ public class ProxyServerTest { private static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode, Payload payload) { return new RawConfig(config.getKey(), config.getDefMd5(), payload, config.getConfigMd5(), - config.getGeneration() + 1, errorCode, config.getDefContent(), Optional.empty()); + config.getGeneration() + 1, false, + errorCode, config.getDefContent(), Optional.empty()); } } diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 342b2b497bb..24e4623372e 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -164,7 +164,7 @@ GetConfig::Main() break; case 'h': retval = 0; - //@fallthrough@ + [[fallthrough]]; case '?': default: usage(); diff --git a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp index 86b5a4a13ba..a88d7deb79a 100644 --- a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp +++ b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp @@ -99,7 +99,7 @@ PingProxy::Main() case '?': default: retval = 1; - // fallthrough + [[fallthrough]]; case 'h': usage(); return retval; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java index bb29e329121..c0eb3e98157 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java @@ -11,7 +11,6 @@ import com.yahoo.config.subscription.impl.ConfigSubscription; * * @param <T> the type of the config * @author vegardh - * @since 5.1 */ public class ConfigHandle<T extends ConfigInstance> { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java index 18d13aac12c..cce3ce0a0c5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.*; /** * @author gjoranv - * @since 5.1.6 */ public class ConfigInstanceUtil { @@ -34,8 +33,8 @@ public class ConfigInstanceUtil { setter.invoke(destination, source); setter.setAccessible(false); } catch (Exception e) { - throw new ConfigurationRuntimeException("Could not set values on config builder." - + destination.getClass().getName(), e); + throw new ConfigurationRuntimeException("Could not set values on config builder." + + destination.getClass().getName(), e); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java index 886d8994d15..d0eda9d27cc 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -3,9 +3,8 @@ package com.yahoo.config.subscription; /** * A type of source of config - * @author vegardh - * @since 5.1 * + * @author vegardh */ public interface ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index bb8d5a87916..c799186435c 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -14,10 +14,10 @@ import java.util.logging.Logger; * Two sets are said to be equal if they contain the same sources, independent of order, * upper/lower-casing and whitespaces. * - * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + * @author gjoranv */ -public class ConfigSourceSet implements ConfigSource -{ +public class ConfigSourceSet implements ConfigSource { + private static final Logger log = Logger.getLogger(ConfigSourceSet.class.getName()); private final Set<String> sources = new LinkedHashSet<>(); @@ -124,4 +124,5 @@ public class ConfigSourceSet implements ConfigSource } return sourceSet; } + } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index b7c45de9f5b..47048fa16c7 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.config.TimingValues; * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. * * @author vegardh - * @since 5.1 */ public class ConfigSubscriber { @@ -32,8 +31,13 @@ public class ConfigSubscriber { private State state = State.OPEN; protected List<ConfigHandle<? extends ConfigInstance>> subscriptionHandles = new ArrayList<>(); private final ConfigSource source; + + /** The last complete config generation received by this */ private long generation = -1; + /** Whether the last generation received was due to a system-internal redeploy, not an application package change */ + private boolean internalRedeploy = false; + /** * Reuse requesters for equal source sets, limit number if many subscriptions. */ @@ -217,10 +221,11 @@ public class ConfigSubscriber { /** * Acquire a snapshot of all configs with the same generation within a timeout. + * * @param timeoutInMillis timeout to wait in milliseconds * @param requireChange if set, at least one config have to change * @return true, if a new config generation has been found for all configs (additionally requires - * that at lest one of them has changed if <code>requireChange</code> is true), false otherwise + * that at lest one of them has changed if <code>requireChange</code> is true), false otherwise */ private boolean acquireSnapshot(long timeoutInMillis, boolean requireChange) { if (state == State.CLOSED) return false; @@ -235,20 +240,22 @@ public class ConfigSubscriber { h.setChanged(false); // Reset this flag, if it was set, the user should have acted on it the last time this method returned true. } boolean reconfigDue; + boolean internalRedeployOnly = true; do { // Keep on polling the subscriptions until we have a new generation across the board, or it times out for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { ConfigSubscription<? extends ConfigInstance> subscription = h.subscription(); - if (!subscription.nextConfig(timeLeftMillis)) { + if ( ! subscription.nextConfig(timeLeftMillis)) { // This subscriber has no new state and we know it has exhausted all time return false; } throwIfExceptionSet(subscription); ConfigSubscription.ConfigState<? extends ConfigInstance> config = subscription.getConfigState(); if (currentGen == null) currentGen = config.getGeneration(); - if (!currentGen.equals(config.getGeneration())) allGenerationsTheSame = false; + if ( ! currentGen.equals(config.getGeneration())) allGenerationsTheSame = false; allGenerationsChanged = allGenerationsChanged && config.isGenerationChanged(); if (config.isConfigChanged()) anyConfigChanged = true; + internalRedeployOnly = internalRedeployOnly && config.isInternalRedeploy(); timeLeftMillis = timeLeftMillis - (System.currentTimeMillis() - started); } reconfigDue = (anyConfigChanged || !requireChange) && allGenerationsChanged && allGenerationsTheSame; @@ -260,6 +267,7 @@ public class ConfigSubscriber { // This indicates the clients will possibly reconfigure their services, so "reset" changed-logic in subscriptions. // Also if appropriate update the changed flag on the handler, which clients use. markSubsChangedSeen(currentGen); + internalRedeploy = internalRedeployOnly; generation = currentGen; } return reconfigDue; @@ -423,6 +431,12 @@ public class ConfigSubscriber { } /** + * Whether the current config generation received by this was due to a system-internal redeploy, + * not an application package change + */ + public boolean isInternalRedeploy() { return internalRedeploy; } + + /** * Convenience interface for clients who only subscribe to one config. Implement this, and pass it to {@link ConfigSubscriber#subscribe(SingleSubscriber, Class, String)}. * * @author vegardh diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java index f9d79561c1c..d68b503eb74 100644 --- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -3,11 +3,11 @@ package com.yahoo.config.subscription; /** * Source specifying raw config, where payload is given programmatically - * @author vegardh - * @since 5.1 * + * @author vegardh */ public class RawSource implements ConfigSource { + public final String payload; /** diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index 02c455bf1c3..76241c560e4 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -23,9 +23,9 @@ import com.yahoo.vespa.config.protocol.DefContent; * Represents one active subscription to one config * * @author vegardh - * @since 5.1 */ public abstract class ConfigSubscription<T extends ConfigInstance> { + protected static Logger log = Logger.getLogger(ConfigSubscription.class.getName()); protected final ConfigSubscriber subscriber; private final AtomicReference<ConfigState<T>> config = new AtomicReference<>(); @@ -35,34 +35,49 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { private State state = State.OPEN; public static class ConfigState<T extends ConfigInstance> { + private final boolean configChanged; private final boolean generationChanged; private final T config; private final Long generation; - private ConfigState(boolean generationChanged, Long generation, boolean configChanged, T config) { - this.configChanged = configChanged; - this.config = config; + private final boolean internalRedeploy; + + private ConfigState(boolean generationChanged, Long generation, boolean internalRedeploy, boolean configChanged, T config) { this.generationChanged = generationChanged; this.generation = generation; + this.internalRedeploy = internalRedeploy; + this.configChanged = configChanged; + this.config = config; } + private ConfigState(Long generation, T config) { - this(false, generation,false, config); + this(false, generation, false, false, config); } + private ConfigState() { - this(false, 0L, false, null); + this(false, 0L, false, false, null); } + private ConfigState<T> createUnchanged() { return new ConfigState<T>(generation, config); } public boolean isConfigChanged() { return configChanged; } public boolean isGenerationChanged() { return generationChanged; } public Long getGeneration() { return generation; } + + /** + * Returns whether this config generation was caused by a system-internal redeploy, + * not an application package change + */ + public boolean isInternalRedeploy() { return internalRedeploy; } + public T getConfig() { return config; } + } + /** * If non-null: The user has set this generation explicitly. nextConfig should take this into account. * Access to these variables _must_ be synchronized, as nextConfig and reload() is likely to be run from * independent threads. */ - private final AtomicReference<Long> reloadedGeneration = new AtomicReference<>(); enum State { @@ -168,30 +183,28 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { } void setConfig(Long generation, T config) { - this.config.set(new ConfigState<>(true, generation, true, config)); + this.config.set(new ConfigState<>(true, generation, false, true, config)); } // Only used by {@link FileConfigSubscription} protected void setConfigIncGen(T config) { ConfigState<T> prev = this.config.get(); - setConfig(prev.getGeneration() + 1, config); + this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, prev.isInternalRedeploy(), true, config)); } - // Only used by {@link FileConfigSubscription} and {@link ConfigSetSubscription} - protected void setConfigIfChangedIncGen(T config) { - ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration() + 1, - !config.equals(prev.getConfig()), config)); - } protected void setConfigIfChanged(T config) { ConfigState<T> prev = this.config.get(); - this.config.set(new ConfigState<>(true, prev.getGeneration(), - !config.equals(prev.getConfig()), config)); + this.config.set(new ConfigState<>(true, prev.getGeneration(), prev.isInternalRedeploy(), !config.equals(prev.getConfig()), config)); } void setGeneration(Long generation) { - ConfigState<T> c = config.get(); - this.config.set(new ConfigState<>(true, generation, c.isConfigChanged(), c.getConfig())); + ConfigState<T> prev = config.get(); + this.config.set(new ConfigState<>(true, generation, prev.isInternalRedeploy(), prev.isConfigChanged(), prev.getConfig())); + } + + void setInternalRedeploy(boolean internalRedeploy) { + ConfigState<T> prev = config.get(); + this.config.set(new ConfigState<>(prev.isGenerationChanged(), prev.getGeneration(), prev.isConfigChanged(), internalRedeploy, prev.getConfig())); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index 8c3f87d0702..bcee06cd667 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -17,9 +17,8 @@ import com.yahoo.log.LogLevel; /** * Subscription used when config id is file:... - * @author vegardh - * @since 5.1 * + * @author vegardh */ public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 068b24cdf1a..69c57120577 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; /** * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. + * * @author vegardh * */ diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 3eea87ab5ba..d95b11533d1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -22,13 +22,14 @@ import com.yahoo.vespa.config.protocol.Payload; * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server * * @author vegardh - * @since 5.1 */ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private JRTConfigRequester requester; private TimingValues timingValues; + // Last time we got an OK JRT callback for this - private long lastOK=0; + private long lastOK = 0; /** * The queue containing either nothing or the one (newest) request that has got callback from JRT, @@ -67,8 +68,11 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } /** - * Polls the callback queue and <em>maybe</em> sets the following (caller must check): generation, generation changed, config, config changed - * Important: it never <em>resets</em> those flags, we must persist that state until the {@link ConfigSubscriber} clears it + * Polls the callback queue and <em>maybe</em> sets the following (caller must check): + * generation, generation changed, config, config changed + * Important: it never <em>resets</em> those flags, we must persist that state until the + * {@link ConfigSubscriber} clears it + * * @param timeoutMillis timeout when polling (returns after at most this time) * @return true if it got anything off the queue and <em>maybe</em> changed any state, false if timed out taking from queue */ @@ -85,6 +89,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc return false; } if (jrtReq.hasUpdatedGeneration()) { + setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { setNewConfig(jrtReq); } else { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index 4870d69bb23..1b2daba32a4 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -22,10 +22,9 @@ import com.yahoo.vespa.config.ConfigPayload; * * @author vegardh * @author gjoranv - * @since 5.1 - * */ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private final String jarName; private final String path; private ZipEntry zipEntry = null; diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index cb623437dba..9ad8f5c6ba2 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -111,12 +111,14 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co } static class OKResponseHandler extends AbstractResponseHandler { + protected void createResponse() { JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); Payload payload = Payload.from(ConfigPayload.empty()); long generation = 1; - jrtReq.addOkResponse(payload, generation, ConfigUtils.getMd5(payload.getData())); + jrtReq.addOkResponse(payload, generation, false, ConfigUtils.getMd5(payload.getData())); } + } public interface ResponseHandler extends Runnable { @@ -131,6 +133,7 @@ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Co } public abstract static class AbstractResponseHandler implements ResponseHandler { + private RequestWaiter requestWaiter; protected Request request; diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index 0d3f04361aa..60811dd38d1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -16,10 +16,9 @@ import com.yahoo.vespa.config.ConfigPayload; * Config is the actual text given after the config id, with newlines * * @author vegardh - * @since 5.1 - * */ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + final String inputPayload; String payload; diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java index 75c5161103e..ae4431e5195 100755 --- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -31,6 +31,7 @@ public class RawConfig extends ConfigInstance { private final String configMd5; private final Optional<VespaVersion> vespaVersion; private long generation; + private boolean internalRedeploy; /** * Constructor for an empty config (not yet resolved). @@ -38,27 +39,29 @@ public class RawConfig extends ConfigInstance { * @param defMd5 The md5 sum of the .def-file. */ public RawConfig(ConfigKey<?> key, String defMd5) { - this(key, defMd5, null, "", 0L, 0, Collections.emptyList(), Optional.empty()); + this(key, defMd5, null, "", 0L, false, 0, Collections.emptyList(), Optional.empty()); } public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, - List<String> defContent, Optional<VespaVersion> vespaVersion) { - this(key, defMd5, payload, configMd5, generation, 0, defContent, vespaVersion); + boolean internalRedeploy, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this(key, defMd5, payload, configMd5, generation, internalRedeploy, 0, defContent, vespaVersion); } /** Copy constructor */ public RawConfig(RawConfig rawConfig) { this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5, - rawConfig.generation, rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion()); + rawConfig.generation, rawConfig.internalRedeploy, rawConfig.errorCode, + rawConfig.defContent, rawConfig.getVespaVersion()); } public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, - int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { + boolean internalRedeploy, int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { this.key = key; this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent); this.payload = payload; this.configMd5 = configMd5; this.generation = generation; + this.internalRedeploy = internalRedeploy; this.errorCode = errorCode; this.defContent = defContent; this.vespaVersion = vespaVersion; @@ -69,8 +72,15 @@ public class RawConfig extends ConfigInstance { * @param req a {@link JRTClientConfigRequest} */ public static RawConfig createFromResponseParameters(JRTClientConfigRequest req) { - return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5(), req.getNewPayload(), req.getNewConfigMd5(), - req.getNewGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + return new RawConfig(req.getConfigKey(), + req.getConfigKey().getMd5(), + req.getNewPayload(), + req.getNewConfigMd5(), + req.getNewGeneration(), + req.responseIsInternalRedeploy(), + 0, + req.getDefContent().asList(), + req.getVespaVersion()); } /** @@ -78,58 +88,47 @@ public class RawConfig extends ConfigInstance { * @param req a {@link JRTClientConfigRequest} */ public static RawConfig createFromServerRequest(JRTServerConfigRequest req) { - return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5() , Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), req.getRequestConfigMd5(), - req.getRequestGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + return new RawConfig(req.getConfigKey(), + req.getConfigKey().getMd5() , + Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), + req.getRequestConfigMd5(), + req.getRequestGeneration(), + req.isInternalRedeploy(), + 0, + req.getDefContent().asList(), + req.getVespaVersion()); } - public ConfigKey<?> getKey() { - return key; - } + public ConfigKey<?> getKey() { return key; } - public String getName() { - return key.getName(); - } + public String getName() { return key.getName(); } - public String getNamespace() { - return key.getNamespace(); - } + public String getNamespace() { return key.getNamespace(); } - public String getConfigId() { - return key.getConfigId(); - } + public String getConfigId() { return key.getConfigId(); } - public String getConfigMd5() { - return configMd5; - } + public String getConfigMd5() { return configMd5; } - public String getDefMd5() { - return defMd5; - } + public String getDefMd5() { return defMd5; } - public long getGeneration() { - return generation; - } + public long getGeneration() { return generation; } - public void setGeneration(long generation) { - this.generation = generation; - } + public void setGeneration(long generation) { this.generation = generation; } - public Payload getPayload() { - return payload; - } + /** + * Returns whether this config generation was created by a system internal redeploy, not an + * application package change. + */ + public boolean isInternalRedeploy() { return internalRedeploy; } - public int errorCode() { - return errorCode; - } + public Payload getPayload() { return payload; } - public String getDefNamespace() { - return key.getNamespace(); - } + public int errorCode() { return errorCode; } - public Optional<VespaVersion> getVespaVersion() { - return vespaVersion; - } + public String getDefNamespace() { return key.getNamespace(); } + + public Optional<VespaVersion> getVespaVersion() { return vespaVersion; } /** * Returns true if this config is equal to the config (same payload md5) in the given request. diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java index 5666417a50a..c07be8337fe 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -13,8 +13,7 @@ import java.util.List; * of this must be thread safe, because a response may be cached and, the methods below should be callable * from multiple request handler threads. * - * @author lulf - * @since 5.1.14 + * @author Ulf Lilleengen */ public interface ConfigResponse { @@ -24,6 +23,8 @@ public interface ConfigResponse { long getGeneration(); + boolean isInternalRedeploy(); + String getConfigMd5(); void serialize(OutputStream os, CompressionType uncompressed) throws IOException; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java index 8e554dc45d8..25fecba425c 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -4,12 +4,13 @@ package com.yahoo.vespa.config.protocol; /** * Interface for config requests used by clients. * - * @author lulf - * @since 5.3 + * @author Ulf Lilleengen */ public interface JRTClientConfigRequest extends JRTConfigRequest { + /** * Validate config response given by the server. If none is given, or an error occurred, this should return false. + * * @return true if valid response, false if not. */ boolean validateResponse(); @@ -17,19 +18,22 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { /** * Test whether ot not the returned config has an updated generation. This should return false if no response have * been given. + * * @return true if generation is updated, false if not. */ boolean hasUpdatedGeneration(); /** * Return the payload in the response given by the server. The payload will be empty if no response was given. + * * @return the config payload. */ Payload getNewPayload(); /** - * Create a new {@link JRTClientConfigRequest} based on this request based on the same request parameters, but having - * the timeout changed. + * Create a new {@link JRTClientConfigRequest} based on this request based on the same request parameters, + * but having the timeout changed. + * * @param timeout server timeout of the new request. * @return a new {@link JRTClientConfigRequest} instance. */ @@ -37,44 +41,57 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { /** * Test whether or not the returned request is an error. + * * @return true if error, false if not. */ boolean isError(); /** * Get the generation of the newly provided config. If none has been given, 0 should be returned. + * * @return the new generation. */ long getNewGeneration(); + /** Returns whether this config change is due to an internal change not an application package change */ + boolean responseIsInternalRedeploy(); + /** * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned. + * * @return a config md5. */ String getNewConfigMd5(); /** - * Test whether or not the payload is contained in this response or not. Should return false for error responses as well. + * Test whether or not the payload is contained in this response or not. + * Should return false for error responses as well. + * * @return true if empty, false if not. */ boolean containsPayload(); /** - * Test whether or not the response contains an updated config or not. False if no response has been returned. + * Test whether or not the response contains an updated config or not. + * False if no response has been returned. + * * @return true if config is updated, false if not. */ boolean hasUpdatedConfig(); /** - * Get the {@link Trace} given in the response by the server. The {@link Trace} can be used to add further tracing - * and later printed to provide useful debug info. + * Get the {@link Trace} given in the response by the server. + * The {@link Trace} can be used to add further tracing and later printed to provide useful debug info. + * * @return a {@link Trace}. */ Trace getResponseTrace(); /** * Get config definition content. + * * @return def as lines. */ DefContent getDefContent(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index 3d7c0dc1e80..1c842a4d1b0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -19,8 +19,7 @@ import java.util.Optional; * * See {@link JRTServerConfigRequestV3} for protocol details. * - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { @@ -71,7 +70,10 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { requestData.getVespaVersion()); } - public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { String hostname = ConfigUtils.getCanonicalHostName(); ConfigKey<T> key = sub.getKey(); ConfigSubscription.ConfigState<T> configState = sub.getConfigState(); @@ -88,7 +90,11 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { } - public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + public static JRTClientConfigRequest createFromRaw(RawConfig config, + long serverTimeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { String hostname = ConfigUtils.getCanonicalHostName(); return createWithParams(config.getKey(), DefContent.fromList(config.getDefContent()), diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java index 54607ec4502..763f672a513 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -6,13 +6,14 @@ import com.yahoo.vespa.config.GetConfigRequest; /** * Interface for config requests at the server end point. * - * @author lulf - * @since 5.3 + * @author Ulf Lilleengen */ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigRequest { + /** * Notify this request that its delayed due to no new config being available at this point. The value * provided in this function should be returned when calling {@link #isDelayedResponse()}. + * * @param delayedResponse true if response is delayed, false if not. */ void setDelayedResponse(boolean delayedResponse); @@ -20,6 +21,7 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque /** * Signal error when handling this request. The error should be reflected in the request state and propagated * back to the client. + * * @param errorCode error code, as described in {@link com.yahoo.vespa.config.ErrorCode}. * @param message message to display for this error, typically printed by client. */ @@ -27,33 +29,46 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque /** * Signal that the request was handled and provide return values typically needed by a client. + * * @param payload The config payload that the client should receive. * @param generation The config generation of the given payload. + * @param internalRedeployment whether this payload was generated from an internal redeployment not an + * application package change * @param configMd5 The md5sum of the given payload. */ - void addOkResponse(Payload payload, long generation, String configMd5); + void addOkResponse(Payload payload, long generation, boolean internalRedeployment, String configMd5); /** * Get the current config md5 of the client config. + * * @return a config md5. */ String getRequestConfigMd5(); /** * Get the current config generation of the client config. + * * @return the current config generation. */ long getRequestGeneration(); /** * Check whether or not this request is delayed. + * * @return true if delayed, false if not. */ boolean isDelayedResponse(); /** + * Returns whether the response config was created by a system internal redeploy, not an application + * package change + */ + boolean isInternalRedeploy(); + + /** * Get the request trace for this request. The trace can be used to trace config execution to provide useful * debug info in production environments. + * * @return a {@link Trace} instance. */ Trace getRequestTrace(); @@ -65,4 +80,5 @@ public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigReque * @return A {@link Payload} that satisfies this request format. */ Payload payloadFromResponse(ConfigResponse response); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index b0096444820..f70ebf39a28 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -20,17 +20,21 @@ import java.io.IOException; * The implementation of addOkResponse is optimized for doing as little copying of payload data as possible, ensuring * that we get a lower memory footprint. * - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ +// TODO: Merge with parent public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { + /** Response field */ + private boolean internalRedeploy = false; + protected JRTServerConfigRequestV3(Request request) { super(request); } @Override - public void addOkResponse(Payload payload, long generation, String configMd5) { + public void addOkResponse(Payload payload, long generation, boolean internalRedeploy, String configMd5) { + this.internalRedeploy = internalRedeploy; boolean changedConfig = !configMd5.equals(getRequestConfigMd5()); boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); Payload responsePayload = payload.withCompression(getCompressionType()); @@ -41,6 +45,7 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { addCommonReturnValues(jsonGenerator); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5); setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_INTERNAL_REDEPLOY, internalRedeploy); jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO); if (responsePayload == null) { throw new RuntimeException("Payload is null for ' " + this + ", not able to create response"); @@ -73,7 +78,11 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { return 3; } + @Override + public boolean isInternalRedeploy() { return internalRedeploy; } + public static JRTServerConfigRequestV3 createFromRequest(Request req) { return new JRTServerConfigRequestV3(req); } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java index 19323fb1a85..386757d2afc 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java @@ -16,8 +16,7 @@ import java.util.logging.Logger; * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of * payload encoding and decoding, as well as adding extra request/response fields. * - * @author lulf - * @since 5.18 + * @author Ulf Lilleengen */ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest { @@ -82,6 +81,7 @@ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest .append(",").append(getVespaVersion()).append("'\n"); sb.append("response='").append(getNewConfigMd5()) .append(",").append(getNewGeneration()) + .append(",").append(responseIsInternalRedeploy()) .append("'\n"); return sb.toString(); } @@ -203,6 +203,11 @@ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest } @Override + public boolean responseIsInternalRedeploy() { + return responseData.getResponseInternalRedeployment(); + } + + @Override public long getRequestGeneration() { return requestData.getRequestGeneration(); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java index e7aae3251d0..566e3597269 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -15,8 +15,7 @@ import java.util.List; /** * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SlimeConfigResponse implements ConfigResponse { @@ -24,17 +23,27 @@ public class SlimeConfigResponse implements ConfigResponse { private final CompressionInfo compressionInfo; private final InnerCNode targetDef; private final long generation; + private final boolean internalRedeploy; private final String configMd5; - public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, String configMd5) { + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, + boolean internalRedeploy, String configMd5) { Utf8Array data = payload.toUtf8Array(true); - return new SlimeConfigResponse(data, targetDef, generation, configMd5, CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); + return new SlimeConfigResponse(data, targetDef, generation, internalRedeploy, + configMd5, + CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); } - public SlimeConfigResponse(Utf8Array payload, InnerCNode targetDef, long generation, String configMd5, CompressionInfo compressionInfo) { + public SlimeConfigResponse(Utf8Array payload, + InnerCNode targetDef, + long generation, + boolean internalRedeploy, + String configMd5, + CompressionInfo compressionInfo) { this.payload = payload; this.targetDef = targetDef; this.generation = generation; + this.internalRedeploy = internalRedeploy; this.configMd5 = configMd5; this.compressionInfo = compressionInfo; } @@ -62,6 +71,13 @@ public class SlimeConfigResponse implements ConfigResponse { return generation; } + /** + * Returns whether this application instance was produced by an internal redeployment, + * not an application package change + */ + @Override + public boolean isInternalRedeploy() { return internalRedeploy; } + @Override public String getConfigMd5() { return configMd5; @@ -81,4 +97,5 @@ public class SlimeConfigResponse implements ConfigResponse { @Override public CompressionInfo getCompressionInfo() { return compressionInfo; } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java index 04f6df7a45b..6add29074d1 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java @@ -11,10 +11,10 @@ import com.yahoo.text.Utf8; * Contains response data for a slime response and methods for decoding the response data that * are common to all {@link Slime} based config requests. * - * @author lulf - * @since 5.18 + * @author Ulf Lilleengen */ class SlimeResponseData { + static final String RESPONSE_VERSION = "version"; static final String RESPONSE_DEF_NAME = "defName"; static final String RESPONSE_DEF_NAMESPACE = "defNamespace"; @@ -24,6 +24,7 @@ class SlimeResponseData { static final String RESPONSE_TRACE = "trace"; static final String RESPONSE_CONFIG_MD5 = "configMD5"; static final String RESPONSE_CONFIG_GENERATION = "generation"; + static final String RESPONSE_INTERNAL_REDEPLOY = "internalRedeploy"; static final String RESPONSE_COMPRESSION_INFO = "compressionInfo"; private final Request request; @@ -66,4 +67,10 @@ class SlimeResponseData { CompressionInfo getCompressionInfo() { return CompressionInfo.fromSlime(getResponseField(RESPONSE_COMPRESSION_INFO)); } + + boolean getResponseInternalRedeployment() { + Inspector inspector = getResponseField(RESPONSE_INTERNAL_REDEPLOY); + return inspector.valid() ? inspector.asBool() : false; + } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java index 662eacc4eea..41bf38ef1dc 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java @@ -19,8 +19,7 @@ import java.util.logging.Logger; * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3 * config protocol. * - * @author lulf - * @since 5.18 + * @author Ulf Lilleengen */ abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { @@ -164,6 +163,10 @@ abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { jsonGenerator.writeNumberField(fieldName, value); } + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException { + jsonGenerator.writeBooleanField(fieldName, value); + } + @Override public long getRequestGeneration() { return requestData.getRequestGeneration(); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java index b9787f76e0c..731a5f50816 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -39,7 +39,7 @@ public class ConfigGetterTest { assertThat(config, is(serviceConfig)); } -@Test + @Test public void testGetFromRawSource() { ConfigGetter<AppConfig> getter = new ConfigGetter<>(new RawSource("message \"one\""), AppConfig.class); AppConfig config = getter.getConfig("test"); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java index 8acab56d838..d17a2ff61f4 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -85,7 +85,7 @@ public class ConfigSetSubscriptionTest { assertEquals(hA1.getConfig().times(), 89); assertEquals(hS.getConfig().stringVal(), "StringVal"); - //Reconfigure all configs, generation should change + // Reconfigure all configs, generation should change a0builder.message("A new message, 0").times(880); a1builder.message("A new message, 1").times(890); barBuilder.stringVal("new StringVal"); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java index 586ad7857f4..3aa422eb116 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -3,6 +3,7 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.foo.AppConfig; import com.yahoo.config.subscription.impl.ConfigSubscription; diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index 33560ff666d..b19da2c1689 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -60,14 +60,14 @@ public class RawConfigTest { assertThat(config.hashCode(), is(not(new RawConfig(key, "a").hashCode()))); // different def md5 // different generation - config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); - RawConfig config2 = new RawConfig(key, defMd5, payload, configMd5, 2L, defContent, Optional.empty()); + config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); + RawConfig config2 = new RawConfig(key, defMd5, payload, configMd5, 2L, false, defContent, Optional.empty()); assertThat(config, is(not(config2))); assertThat(config.hashCode(), is(not(config2.hashCode()))); // different config md5 and with vespa version final VespaVersion vespaVersion = VespaVersion.fromString("5.37.38"); - RawConfig config3 = new RawConfig(key, defMd5, payload, "9999", generation, defContent, Optional.of(vespaVersion)); + RawConfig config3 = new RawConfig(key, defMd5, payload, "9999", generation, false, defContent, Optional.of(vespaVersion)); assertThat(config, is(not(config3))); assertThat(config.hashCode(), is(not(config3.hashCode()))); // Check that vespa version is set correctly @@ -81,19 +81,19 @@ public class RawConfigTest { assertFalse(config.equals(key)); // errors - RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, configMd5, generation, 1, defContent, Optional.empty()); + RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 1, defContent, Optional.empty()); assertThat(errorConfig1, is(errorConfig1)); assertThat(config, is(not(errorConfig1))); assertThat(config.hashCode(), is(not(errorConfig1.hashCode()))); assertThat(errorConfig1, is(errorConfig1)); - RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, configMd5, generation, 2, defContent, Optional.empty()); + RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, configMd5, generation, false, 2, defContent, Optional.empty()); assertThat(errorConfig1, is(not(errorConfig2))); assertThat(errorConfig1.hashCode(), is(not(errorConfig2.hashCode()))); } @Test public void payload() { - RawConfig config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); + RawConfig config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); assertThat(config.getPayload(), is(payload)); assertThat(config.getConfigMd5(), is(configMd5)); assertThat(config.getGeneration(), is(generation)); @@ -104,19 +104,19 @@ public class RawConfigTest { public void require_correct_defmd5() { final String defMd5ForEmptyDefContent = "d41d8cd98f00b204e9800998ecf8427e"; - RawConfig config = new RawConfig(key, null, payload, configMd5, generation, defContent, Optional.empty()); + RawConfig config = new RawConfig(key, null, payload, configMd5, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, "", payload, configMd5, generation, defContent, Optional.empty()); + config = new RawConfig(key, "", payload, configMd5, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); + config = new RawConfig(key, defMd5, payload, configMd5, generation, false, defContent, Optional.empty()); assertThat(config.getDefMd5(), is(defMd5)); - config = new RawConfig(key, null, payload, configMd5, generation, null, Optional.empty()); + config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty()); assertNull(config.getDefMd5()); - config = new RawConfig(key, null, payload, configMd5, generation, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, null, payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); - config = new RawConfig(key, "", payload, configMd5, generation, null, Optional.empty()); + config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty()); assertThat(config.getDefMd5(), is("")); - config = new RawConfig(key, "", payload, configMd5, generation, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, "", payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); } diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java index 9a7b216d6b0..91adc544d88 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java @@ -20,8 +20,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigResponseTest { @@ -30,7 +29,7 @@ public class ConfigResponseTest { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); InnerCNode targetDef = dParser.getTree(); - ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, "mymd5"); + ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, false, "mymd5"); List<String> payload = response.getLegacyPayload(); assertNotNull(payload); assertThat(payload.size(), is(6)); @@ -45,13 +44,12 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_decompresses_on_serialize() throws IOException { - ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); InnerCNode targetDef = dParser.getTree(); Utf8Array data = configPayload.toUtf8Array(true); Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.getBytes())); - ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); + ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); List<String> payload = response.getLegacyPayload(); assertNotNull(payload); assertThat(payload.size(), is(6)); @@ -63,4 +61,5 @@ public class ConfigResponseTest { response.serialize(baos, CompressionType.UNCOMPRESSED); assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } + } diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java index 1f5274b832b..cdaaf2061f4 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java @@ -125,7 +125,7 @@ public abstract class JRTConfigRequestBase { @Test public void next_request_when_error_is_correct() { - serverReq.addOkResponse(createPayload(), 999999, "newmd5"); + serverReq.addOkResponse(createPayload(), 999999, false, "newmd5"); serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message"); System.out.println(serverReq); JRTClientConfigRequest next = clientReq.nextRequest(6); @@ -141,7 +141,7 @@ public abstract class JRTConfigRequestBase { Payload payload = createPayload("vale"); String md5 = ConfigUtils.getMd5(payload.getData()); long generation = 4L; - serverReq.addOkResponse(payload, generation, md5); + serverReq.addOkResponse(payload, generation, false, md5); assertTrue(clientReq.validateResponse()); assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString())); assertThat(clientReq.getNewGeneration(), is(4L)); @@ -168,7 +168,7 @@ public abstract class JRTConfigRequestBase { @Test public void generation_only_is_updated() { Payload payload = createPayload(); - serverReq.addOkResponse(payload, 4L, ConfigUtils.getMd5(payload.getData())); + serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData())); boolean value = clientReq.validateResponse(); assertTrue(clientReq.errorMessage(), value); assertFalse(clientReq.hasUpdatedConfig()); @@ -188,7 +188,7 @@ public abstract class JRTConfigRequestBase { @Test public void nothing_is_updated() { Payload payload = createPayload(); - serverReq.addOkResponse(payload, currentGeneration, configMd5); + serverReq.addOkResponse(payload, currentGeneration, false, configMd5); assertTrue(clientReq.validateResponse()); assertFalse(clientReq.hasUpdatedConfig()); assertFalse(clientReq.hasUpdatedGeneration()); @@ -199,7 +199,7 @@ public abstract class JRTConfigRequestBase { Payload payload = Payload.from(ConfigPayload.empty()); clientReq = createReq(payload); serverReq = createReq(clientReq.getRequest()); - serverReq.addOkResponse(payload, currentGeneration, ConfigUtils.getMd5(payload.getData())); + serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData())); boolean val = clientReq.validateResponse(); assertTrue(clientReq.errorMessage(), val); assertFalse(clientReq.hasUpdatedConfig()); @@ -238,7 +238,7 @@ public abstract class JRTConfigRequestBase { @Override protected void createResponse() { JRTServerConfigRequest serverRequest = createReq(request); - serverRequest.addOkResponse(createPayload(), currentGeneration, configMd5); + serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5); } }); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index ccb68da5f85..d1879c9318b 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.Arrays; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -70,10 +71,11 @@ public class JRTConfigRequestV3Test extends JRTConfigRequestBase { @Test public void emptypayload() { ConfigPayload payload = ConfigPayload.empty(); - SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, ConfigUtils.getMd5(payload)); - serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), response.getConfigMd5()); + SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, false, ConfigUtils.getMd5(payload)); + serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5()); assertTrue(clientReq.validateResponse()); assertTrue(clientReq.hasUpdatedGeneration()); assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is("{}")); + assertFalse(clientReq.responseIsInternalRedeploy()); } } diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java new file mode 100644 index 00000000000..8c0e276b3d3 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 281db6fb43d..281db6fb43d 100644 --- a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 7a1ddeb8f66..bf4c9599f4a 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -13,10 +13,13 @@ zookeeperserver[].port int default=2181 zookeeper.barrierTimeout long default=120 # in seconds zookeeperLocalhostAffinity bool default=true -# Misc +# Directories configModelPluginDir[] string configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" +fileReferencesDir string default="var/db/vespa/filedistribution/" + +# Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 multitenant bool default=false @@ -52,5 +55,6 @@ ztsUrl string default="" nodeAdminInContainer bool default=true # Maintainers +maintainerIntervalMinutes int default=30 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 diff --git a/configgen/pom.xml b/configgen/pom.xml index e8de114f8d5..be9cd733601 100644 --- a/configgen/pom.xml +++ b/configgen/pom.xml @@ -19,14 +19,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - </dependency> </dependencies> <build> <plugins> @@ -63,32 +55,6 @@ <updateReleaseInfo>true</updateReleaseInfo> </configuration> </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>compile</id> - <goals> - <goal>compile</goal> - </goals> - <phase>compile</phase> - </execution> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - <execution> - <phase>process-resources</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java new file mode 100644 index 00000000000..bf3fc2902a1 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java @@ -0,0 +1,351 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import com.yahoo.config.codegen.LeafCNode.FileLeaf; +import com.yahoo.config.codegen.LeafCNode.PathLeaf; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.ConfigGenerator.boxedDataType; +import static com.yahoo.config.codegen.ConfigGenerator.indentCode; +import static com.yahoo.config.codegen.ConfigGenerator.nodeClass; +import static com.yahoo.config.codegen.ConfigGenerator.userDataType; +import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION; +import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; +import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX; +import static java.util.Arrays.stream; + +/** + * @author gjoranv + * @author ollivir + */ + +public class BuilderGenerator { + public static String getBuilder(InnerCNode node) { + return getDeclaration(node) + "\n" + // + indentCode(INDENTATION, getUninitializedScalars(node) + "\n\n" + // + stream(node.getChildren()).map(BuilderGenerator::getBuilderFieldDefinition).collect(Collectors.joining("\n")) + + "\n\n" + // + getBuilderConstructors(node, nodeClass(node)) + "\n\n" + // + getOverrideMethod(node) + "\n\n" + // + getBuilderSetters(node) + "\n" + // + getSpecialRootBuilderCode(node)) + + "}"; + } + + private static String getDeclaration(InnerCNode node) { + String getInterfaces = (node.getParent() == null) ? "implements ConfigInstance.Builder" : "implements ConfigBuilder"; + + return "public static class Builder " + getInterfaces + " {"; + } + + private static String getSpecialRootBuilderCode(InnerCNode node) { + return (node.getParent() == null) ? "\n" + getDispatchCode() + "\n" : ""; + } + + private static String getDispatchCode() { + // Use full path to @Override, as users are free to define an inner node called + // 'override'. (summarymap.def does) + // The generated inner 'Override' class would otherwise be mistaken for the + // annotation. + return "@java.lang.Override\n" + // + "public final boolean dispatchGetConfig(ConfigInstance.Producer producer) {\n" + // + " if (producer instanceof Producer) {\n" + // + " ((Producer)producer).getConfig(this);\n" + // + " return true;\n" + // + " }\n" + // + " return false;\n" + // + "}\n" + // + "\n" + // + "@java.lang.Override\n" + // + "public final String getDefMd5() { return CONFIG_DEF_MD5; }\n" + // + "@java.lang.Override\n" + // + "public final String getDefName() { return CONFIG_DEF_NAME; }\n" + // + "@java.lang.Override\n" + // + "public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }"; + } + + private static String getUninitializedScalars(InnerCNode node) { + List<String> scalarsWithoutDefault = new ArrayList<>(); + for (CNode child : node.getChildren()) { + if (child instanceof LeafCNode && (!child.isArray && !child.isMap && ((LeafCNode) child).getDefaultValue() == null)) { + scalarsWithoutDefault.add("\"" + child.getName() + "\""); + } + } + + String uninitializedList = (scalarsWithoutDefault.size() > 0) + ? "Arrays.asList(\n" + indentCode(INDENTATION, String.join(",\n", scalarsWithoutDefault) + "\n)") + : ""; + + return "private Set<String> " + INTERNAL_PREFIX + "uninitialized = new HashSet<String>(" + uninitializedList + ");"; + } + + private static String getBuilderFieldDefinition(CNode node) { + if (node.isArray) { + return String.format("public List<%s> %s = new ArrayList<>();", builderType(node), node.getName()); + } else if (node.isMap) { + return String.format("public Map<String, %s> %s = new LinkedHashMap<>();", builderType(node), node.getName()); + } else if (node instanceof InnerCNode) { + return String.format("public %s %s = new %s();", builderType(node), node.getName(), builderType(node)); + } else if (node instanceof LeafCNode) { + return String.format("private %s %s = null;", boxedBuilderType((LeafCNode) node), node.getName()); + } else { + throw new IllegalStateException("Cannot produce builder field definition for node"); // Should not happen + } + } + + private static String getBuilderSetters(CNode node) { + List<String> elem = new ArrayList<>(); + CNode[] children = node.getChildren(); + + for (CNode child : children) { + if (child instanceof InnerCNode && child.isArray) { + elem.add(BuilderSetters.innerArraySetters((InnerCNode) child)); + } else if (child instanceof InnerCNode && child.isMap) { + elem.add(BuilderSetters.innerMapSetters(child)); + } else if (child instanceof LeafCNode && child.isArray) { + elem.add(BuilderSetters.leafArraySetters((LeafCNode) child)); + } else if (child instanceof LeafCNode && child.isMap) { + elem.add(BuilderSetters.leafMapSetters(child)); + } else if (child instanceof InnerCNode) { + elem.add(BuilderSetters.structSetter((InnerCNode) child)); + } else if (child instanceof LeafCNode) { + elem.add(BuilderSetters.scalarSetters((LeafCNode) child)); + } + } + return String.join("\n\n", elem); + } + + private static class BuilderSetters { + private static String structSetter(InnerCNode n) { + return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + // + " " + n.getName() + " = " + INTERNAL_PREFIX + "builder;\n" + // + " return this;\n" + // + "}"; + } + + private static String innerArraySetters(InnerCNode n) { + return "/**\n" + // + " * Add the given builder to this builder's list of " + nodeClass(n) + " builders\n" + // + " * @param " + INTERNAL_PREFIX + "builder a builder\n" + // + " * @return this builder\n" + // + " */\n" + // + "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + // + " " + n.getName() + ".add(" + INTERNAL_PREFIX + "builder);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "/**\n" + // + " * Set the given list as this builder's list of " + nodeClass(n) + " builders\n" + // + " * @param __builders a list of builders\n" + // + " * @return this builder\n" + // + " */\n" + // + "public Builder " + n.getName() + "(List<" + builderType(n) + "> __builders) {\n" + // + " " + n.getName() + " = __builders;\n" + // + " return this;\n" + // + "}"; + } + + private static String publicLeafNodeSetters(LeafCNode n) { + return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "value) {\n" + // + " " + n.getName() + ".add(" + INTERNAL_PREFIX + "value);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "public Builder " + n.getName() + "(Collection<" + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + // + " " + n.getName() + ".addAll(" + INTERNAL_PREFIX + "values);\n" + // + " return this;\n" + // + "}"; + } + + private static String privateLeafNodeSetter(LeafCNode n) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { + return ""; + } else { + return "\n\n" + // + "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "value) {\n" + // + " return " + n.getName() + "(" + builderType(n) + ".valueOf(" + INTERNAL_PREFIX + "value));\n" + // + "}"; + } + } + + private static String leafArraySetters(LeafCNode n) { + return publicLeafNodeSetters(n) + privateLeafNodeSetter(n); + } + + private static String innerMapSetters(CNode n) { + return "public Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, " + builderType(n) + " " + INTERNAL_PREFIX + + "value) {\n" + // + " " + n.getName() + ".put(" + INTERNAL_PREFIX + "key, " + INTERNAL_PREFIX + "value);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "public Builder " + n.getName() + "(Map<String, " + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + // + " " + n.getName() + ".putAll(" + INTERNAL_PREFIX + "values);\n" + // + " return this;\n" + // + "}"; + } + + private static String privateLeafMapSetter(CNode n) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { + return ""; + } else { + return "\n\n" + // + "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, String " + INTERNAL_PREFIX + "value) {\n" + // + " return " + n.getName() + "(" + INTERNAL_PREFIX + "key, " + builderType(n) + ".valueOf(" + INTERNAL_PREFIX + + "value));\n" + // + "}"; + } + } + + private static String leafMapSetters(CNode n) { + return innerMapSetters(n) + privateLeafMapSetter(n); + } + + private static String scalarSetters(LeafCNode n) { + String name = n.getName(); + + String signalInitialized = (n.getDefaultValue() == null) ? " " + INTERNAL_PREFIX + "uninitialized.remove(\"" + name + "\");\n" + : ""; + + String bType = builderType(n); + String stringSetter = "String".equals(bType) || "FileReference".equals(bType) ? "" + : String.format("\nprivate Builder %s(String %svalue) {\n" + // + " return %s(%s.valueOf(%svalue));\n" + // + "}", name, INTERNAL_PREFIX, name, boxedDataType(n), INTERNAL_PREFIX); + + String getNullGuard = bType.equals(boxedBuilderType(n)) ? String.format( + "\nif (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");", INTERNAL_PREFIX) : ""; + + return String.format("public Builder %s(%s %svalue) {%s\n" + // + " %s = %svalue;\n" + // + "%s", name, bType, INTERNAL_PREFIX, getNullGuard, name, INTERNAL_PREFIX, signalInitialized) + // + " return this;" + "\n}\n" + stringSetter; + } + } + + private static String setBuilderValueFromConfig(CNode child, CNode node) { + final String name = child.getName(); + final boolean isArray = child.isArray; + final boolean isMap = child.isMap; + + if (child instanceof FileLeaf && isArray) { + return name + "(" + userDataType(child) + ".toValues(config." + name + "()));"; + } else if (child instanceof FileLeaf && isMap) { + return name + "(" + userDataType(child) + ".toValueMap(config." + name + "()));"; + } else if (child instanceof FileLeaf) { + return name + "(config." + name + "().value());"; + } else if (child instanceof PathLeaf && isArray) { + return name + "(" + nodeClass(child) + ".toFileReferences(config." + name + "));"; + } else if (child instanceof PathLeaf && isMap) { + return name + "(" + nodeClass(child) + ".toFileReferenceMap(config." + name + "));"; + } else if (child instanceof PathLeaf) { + return name + "(config." + name + ".getFileReference());"; + } else if (child instanceof LeafCNode) { + return name + "(config." + name + "());"; + } else if (child instanceof InnerCNode && isArray) { + return setInnerArrayBuildersFromConfig((InnerCNode) child, node); + } else if (child instanceof InnerCNode && isMap) { + return setInnerMapBuildersFromConfig((InnerCNode) child); + } else { + return name + "(new " + builderType(child) + "(config." + name + "()));"; + } + } + + private static String setInnerArrayBuildersFromConfig(InnerCNode innerArr, CNode node) { + final String elemName = createUniqueSymbol(node, innerArr.getName()); + + return "for (" + userDataType(innerArr) + " " + elemName + " : config." + innerArr.getName() + "()) {\n" + // + " " + innerArr.getName() + "(new " + builderType(innerArr) + "(" + elemName + "));\n" + // + "}"; + } + + private static String setInnerMapBuildersFromConfig(InnerCNode innerMap) { + final String entryName = INTERNAL_PREFIX + "entry"; + return "for (Map.Entry<String, " + userDataType(innerMap) + "> " + entryName + " : config." + innerMap.getName() + + "().entrySet()) {\n" + // + " " + innerMap.getName() + "(" + entryName + ".getKey(), new " + userDataType(innerMap) + ".Builder(" + entryName + + ".getValue()));\n" + // + "}"; + } + + private static String getBuilderConstructors(CNode node, String className) { + return "public Builder() { }\n" + // + "\n" + // + "public Builder(" + className + " config) {\n" + // + indentCode(INDENTATION, + stream(node.getChildren()).map(child -> setBuilderValueFromConfig(child, node)).collect(Collectors.joining("\n"))) + + // + "\n}"; + } + + private static String conditionStatement(CNode child) { + final String superior = INTERNAL_PREFIX + "superior"; + + if (child.isArray) { + return "if (!" + superior + "." + child.getName() + ".isEmpty())"; + } else if (child.isMap) { + return ""; + } else if (child instanceof LeafCNode) { + return "if (" + superior + "." + child.getName() + " != null)"; + } else { + return ""; + } + } + + private static String overrideBuilderValue(CNode child) { + final String superior = INTERNAL_PREFIX + "superior"; + final String method = "override"; + final String name = child.getName(); + final String callSetter = name + "(" + superior + "." + name + ");"; + + if (child.isArray) { + String arrayOverride = INDENTATION + name + ".addAll(" + superior + "." + name + ");"; + return conditionStatement(child) + "\n" + arrayOverride; + } else if (child instanceof InnerCNode && !child.isArray && !child.isMap) { + return name + "(" + name + "." + method + "(" + superior + "." + name + "));"; + } else if (child.isMap) { + return callSetter; + } else { + return conditionStatement(child) + "\n" + INDENTATION + callSetter; + } + } + + private static String getOverrideMethod(CNode node) { + final String superior = INTERNAL_PREFIX + "superior"; + final String method = "override"; + + return "private Builder " + method + "(Builder " + superior + ") {\n" + // + indentCode(INDENTATION, + stream(node.getChildren()).map(BuilderGenerator::overrideBuilderValue).collect(Collectors.joining("\n"))) + + "\n" + // + " return this;\n" + // + "}"; + } + + private static String builderType(CNode node) { + if (node instanceof InnerCNode) { + return boxedDataType(node) + ".Builder"; + } else if (node instanceof FileLeaf) { + return "String"; + } else if (node instanceof PathLeaf) { + return "FileReference"; + } else if (node instanceof LeafCNode && (node.isArray || node.isMap)) { + return boxedDataType(node); + } else { + return userDataType(node); + } + } + + private static String boxedBuilderType(LeafCNode node) { + if (node instanceof FileLeaf) { + return "String"; + } else if (node instanceof PathLeaf) { + return "FileReference"; + } else { + return boxedDataType(node); + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java new file mode 100644 index 00000000000..9980cf565b1 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java @@ -0,0 +1,444 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import com.yahoo.config.codegen.LeafCNode.BooleanLeaf; +import com.yahoo.config.codegen.LeafCNode.DoubleLeaf; +import com.yahoo.config.codegen.LeafCNode.EnumLeaf; +import com.yahoo.config.codegen.LeafCNode.FileLeaf; +import com.yahoo.config.codegen.LeafCNode.IntegerLeaf; +import com.yahoo.config.codegen.LeafCNode.LongLeaf; +import com.yahoo.config.codegen.LeafCNode.PathLeaf; +import com.yahoo.config.codegen.LeafCNode.ReferenceLeaf; +import com.yahoo.config.codegen.LeafCNode.StringLeaf; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.BuilderGenerator.getBuilder; +import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION; +import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX; +import static java.util.Arrays.stream; + +/** + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class ConfigGenerator { + // TODO: don't take indent as method param - the caller should indent + public static String generateContent(String indent, InnerCNode node, boolean isOuter) { + CNode[] children = node.getChildren(); + + return indentCode(indent, + getBuilder(node) + "\n\n" + + stream(children).map(ConfigGenerator::getFieldDefinition).collect(Collectors.joining("\n")) + "\n\n" + + getConstructors(node) + "\n\n" + + getAccessors(children) + "\n\n" + + getGetChangesRequiringRestart(node) + "\n\n" + + getContainsFieldsFlaggedWithRestart(node, isOuter) + + getStaticMethods(node) + + generateCodeForChildren(children, indent) + ); + } + + private static String generateCodeForChildren(CNode[] children, String indent) { + List<String> pieces = new LinkedList<>(); + for (CNode child : children) { + if (child instanceof EnumLeaf) { + pieces.add(getEnumCode((EnumLeaf) child) + "\n"); + } else if (child instanceof InnerCNode) { + pieces.add(getInnerDefinition((InnerCNode) child, indent) + "\n"); + } + } + return String.join("\n", pieces); + } + + private static String getInnerDefinition(InnerCNode inner, String indent) { + return (getClassDoc(inner) + "\n" +// + getClassDeclaration(inner) + "\n" +// + generateContent(indent, inner, false)).trim() + "\n}"; + } + + private static String getClassDeclaration(CNode node) { + return "public final static class " + nodeClass(node) + " extends InnerNode { \n"; + } + + private static String getFieldDefinition(CNode node) { + String fieldDef; + if (node instanceof LeafCNode && node.isArray) { + fieldDef = String.format("LeafNodeVector<%s, %s> %s;", boxedDataType(node), nodeClass(node), node.getName()); + } else if (node instanceof InnerCNode && node.isArray) { + fieldDef = String.format("InnerNodeVector<%s> %s;", nodeClass(node), node.getName()); + } else if (node.isMap) { + fieldDef = String.format("Map<String, %s> %s;", nodeClass(node), node.getName()); + } else { + fieldDef = String.format("%s %s;", nodeClass(node), node.getName()); + } + return node.getCommentBlock("//") + "private final " + fieldDef; + } + + private static String getStaticMethods(InnerCNode node) { + if (node.isArray) { + return getStaticMethodsForInnerArray(node) + "\n\n"; + } else if (node.isMap) { + return getStaticMethodsForInnerMap(node) + "\n\n"; + } else { + return ""; + } + } + + private static String getContainsFieldsFlaggedWithRestart(CNode node, boolean isOuter) { + if (isOuter) { + return String.format("private static boolean containsFieldsFlaggedWithRestart() {\n" +// + " return %b;\n" +// + "}\n\n", node.needRestart()); + } else { + return ""; + } + } + + private static String getGetChangesRequiringRestart(InnerCNode node) { + List<String> comparisons = new LinkedList<>(); + for (CNode child : node.getChildren()) { + if (child.needRestart()) { + comparisons.add("\n " + getComparison(child)); + } + } + + return "private ChangesRequiringRestart getChangesRequiringRestart(" + nodeClass(node) + " newConfig) {\n" +// + " ChangesRequiringRestart changes = new ChangesRequiringRestart(\"" + node.getName() + "\");" + String.join("", comparisons) + "\n" +// + " return changes;\n" +// + "}"; + } + + private static String quotedComment(CNode node) { + return node.getComment().replace("\n", "\\n").replace("\"", "\\\""); + } + + private static String getComparison(CNode node) { + if (node instanceof InnerCNode && node.isArray) { + return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));"; + } else if (node instanceof InnerCNode && node.isMap) { + return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));"; + } else if (node instanceof InnerCNode) { + return " changes.mergeChanges(\"" + node.getName() + "\", this." + node.getName() + ".getChangesRequiringRestart(newConfig." + node.getName() + "));"; + } else if (node.isArray) { + return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));"; + } else if (node.isMap) { + return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));"; + } else { + return " changes.compare(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\");"; + } + } + + private static String scalarDefault(LeafCNode scalar) { + if (scalar.getDefaultValue() == null) { + return ""; + } else if (scalar instanceof EnumLeaf && scalar.getDefaultValue().getValue() == null) { + return ""; + } else if (scalar instanceof EnumLeaf) { + return nodeClass(scalar) + "." + scalar.getDefaultValue().getStringRepresentation(); + } else if (scalar instanceof LongLeaf) { + return scalar.getDefaultValue().getStringRepresentation() + "L"; + } else if (scalar instanceof DoubleLeaf) { + return scalar.getDefaultValue().getStringRepresentation() + "D"; + } else { + return scalar.getDefaultValue().getStringRepresentation(); + } + } + + private static String assignFromBuilder(CNode child) { + final String name = child.getName(); + final String className = nodeClass(child); + final boolean isArray = child.isArray; + final boolean isMap = child.isMap; + + if (child instanceof FileLeaf && isArray) { + return name + " = LeafNodeVector.createFileNodeVector(builder." + name + ");"; + } else if (child instanceof PathLeaf && isArray) { + return name + " = LeafNodeVector.createPathNodeVector(builder." + name + ");"; + } else if (child instanceof LeafCNode && isArray) { + return name + " = new LeafNodeVector<>(builder." + name + ", new " + className + "());"; + } else if (child instanceof FileLeaf && isMap) { + return name + " = LeafNodeMaps.asFileNodeMap(builder." + name + ");"; + } else if (child instanceof PathLeaf && isMap) { + return name + " = LeafNodeMaps.asPathNodeMap(builder." + name + ");"; + } else if (child instanceof LeafCNode && isMap) { + return name + " = LeafNodeMaps.asNodeMap(builder." + name + ", new " + className + "());"; + } else if (child instanceof InnerCNode && isArray) { + return name + " = " + className + ".createVector(builder." + name + ");"; + } else if (child instanceof InnerCNode && isMap) { + return name + " = " + className + ".createMap(builder." + name + ");"; + } else if (child instanceof InnerCNode) { + return name + " = new " + className + "(builder." + name + ", throwIfUninitialized);"; + } else if (child instanceof LeafCNode) { + return name + " = (builder." + name + " == null) ?\n" +// + " new " + className + "(" + scalarDefault((LeafCNode) child) + ") : new " + className + "(builder." + name + ");"; + } else { + throw new IllegalStateException("Cannot create assignment for node"); // should not happen + } + } + + private static String getConstructors(InnerCNode inner) { + // TODO: merge these two constructors into one when the config library uses builders to set values from payload. + return "public " + nodeClass(inner) + "(Builder builder) {\n" +// + " this(builder, true);\n" +// + "}\n" +// + "\n" +// + "private " + nodeClass(inner) + "(Builder builder, boolean throwIfUninitialized) {\n" +// + " if (throwIfUninitialized && ! builder." + INTERNAL_PREFIX + "uninitialized.isEmpty())\n" +// + " throw new IllegalArgumentException(\"The following builder parameters for \" +\n" +// + " \"" + inner.getFullName() + " must be initialized: \" + builder." + INTERNAL_PREFIX + "uninitialized);\n" +// + "\n" +// + indentCode(INDENTATION, stream(inner.getChildren()).map(ConfigGenerator::assignFromBuilder).collect(Collectors.joining("\n"))) + "\n" +// + "}"; + } + + private static String getAccessorCode(CNode node) { + if (node.isArray) { + return accessorsForArray(node); + } else if (node.isMap) { + return accessorsForMap(node); + } else { + return accessorForStructOrScalar(node); + } + } + + private static String valueAccessor(CNode node) { + if (node instanceof LeafCNode) { + return ".value()"; + } else { + return ""; + } + } + + private static String listAccessor(CNode node) { + if (node instanceof LeafCNode) { + return node.getName() + ".asList()"; + } else { + return node.getName(); + } + } + + private static String mapAccessor(CNode node) { + if (node instanceof LeafCNode) { + return "LeafNodeMaps.asValueMap(" + node.getName() + ")"; + } else { + return "Collections.unmodifiableMap(" + node.getName() + ")"; + } + } + + private static String accessorsForArray(CNode node) { + final String name = node.getName(); + final String fullName = node.getFullName(); + return "/**\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public List<" + boxedDataType(node) + "> " + name + "() {\n" +// + " return " + listAccessor(node) + ";\n" +// + "}\n" +// + "\n" +// + "/**\n" +// + " * @param i the index of the value to return\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + name + "(int i) {\n" +// + " return " + name + ".get(i)" + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String accessorsForMap(CNode node) { + final String name = node.getName(); + final String fullName = node.getFullName(); + + return "/**\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public Map<String, " + boxedDataType(node) + "> " + name + "() {\n" +// + " return " + mapAccessor(node) + ";\n" +// + "}\n" +// + "\n" +// + "/**\n" +// + " * @param key the key of the value to return\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + name + "(String key) {\n" +// + " return " + name + ".get(key)" + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String accessorForStructOrScalar(CNode node) { + return "/**\n" +// + " * @return " + node.getFullName() + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + node.getName() + "() {\n" +// + " return " + node.getName() + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String getAccessors(CNode[] children) { + List<String> accessors = new LinkedList<>(); + for (CNode child : children) { + String accessor = getAccessorCode(child); + if (accessor.isEmpty() == false) { + accessors.add(accessor); + } + } + return String.join("\n\n", accessors); + } + + private static String getStaticMethodsForInnerArray(InnerCNode inner) { + final String nc = nodeClass(inner); + return String.format("private static InnerNodeVector<%s> createVector(List<Builder> builders) {\n" +// + " List<%s> elems = new ArrayList<>();\n" +// + " for (Builder b : builders) {\n" +// + " elems.add(new %s(b));\n" +// + " }\n" +// + " return new InnerNodeVector<%s>(elems);\n" +// + "}", nc, nc, nc, nc); + } + + private static String getStaticMethodsForInnerMap(InnerCNode inner) { + final String nc = nodeClass(inner); + return String.format( + "private static Map<String, %s> createMap(Map<String, Builder> builders) {\n" +// + " Map<String, %s> ret = new LinkedHashMap<>();\n" +// + " for(String key : builders.keySet()) {\n" +// + " ret.put(key, new %s(builders.get(key)));\n" +// + " }\n" +// + " return Collections.unmodifiableMap(ret);\n" +// + "}", nc, nc, nc); + } + + private static String getEnumCode(EnumLeaf en) { + String enumValues = stream(en.getLegalValues()).map(e -> String.format(" public final static Enum %s = Enum.%s;", e, e)).collect(Collectors.joining("\n")); + + String code = String.format("%s\n" +// + "public final static class %s extends EnumNode<%s> {\n" +// + "\n" +// + " public %s(){\n" +// + " this.value = null;\n" +// + " }\n" +// + "\n" +// + " public %s(Enum enumValue) {\n" +// + " super(enumValue != null);\n" +// + " this.value = enumValue;\n" +// + " }\n" +// + "\n" +// + " public enum Enum {%s}\n" +// + "%s\n" +// + "\n" +// + " @Override\n" +// + " protected boolean doSetValue(@NonNull String name) {\n" +// + " try {\n" +// + " value = Enum.valueOf(name);\n" +// + " return true;\n" +// + " } catch (IllegalArgumentException e) {\n" +// + " }\n" +// + " return false;\n" +// + " }\n" +// + "}", getClassDoc(en), + nodeClass(en), + nodeClass(en) + ".Enum", + nodeClass(en), + nodeClass(en), + String.join(", ", en.getLegalValues()), + enumValues); + + return indentCode("", code); + } + + private static String getClassDoc(CNode node) { + String header = "/**\n" + " * This class represents " + node.getFullName(); + String nodeComment = node.getCommentBlock(" *"); + if (nodeComment.isEmpty()) { + return header + "\n */"; + } else { + if (nodeComment.endsWith("\n")) { + nodeComment = nodeComment.substring(0, nodeComment.length() - 1); + } + return header + "\n * \n" + nodeComment + "\n */"; + } + } + + static String indentCode(String indent, String code) { + List<String> indented = new LinkedList<>(); + for (String line : code.split("\n", -1)) { + indented.add(line.length() > 0 ? indent + line : line); + } + return String.join("\n", indented); + } + + /** + * @return the name of the class that is generated by this node. + */ + static String nodeClass(CNode node) { + if (node.getName().length() == 0) { + throw new CodegenRuntimeException("Node with empty name, under parent " + node.getParent().getName()); + } else if (node instanceof InnerCNode && node.getParent() == null) { + return ConfiggenUtil.createClassName(node.getName()); + } else if (node instanceof BooleanLeaf) { + return "BooleanNode"; + } else if (node instanceof DoubleLeaf) { + return "DoubleNode"; + } else if (node instanceof FileLeaf) { + return "FileNode"; + } else if (node instanceof PathLeaf) { + return "PathNode"; + } else if (node instanceof IntegerLeaf) { + return "IntegerNode"; + } else if (node instanceof LongLeaf) { + return "LongNode"; + } else if (node instanceof ReferenceLeaf) { + return "ReferenceNode"; + } else if (node instanceof StringLeaf) { + return "StringNode"; + } else { + return ConfiggenUtil.capitalize(node.getName()); + } + } + + static String userDataType(CNode node) { + if (node instanceof InnerCNode) { + return nodeClass(node); + } else if (node instanceof EnumLeaf) { + return nodeClass(node) + ".Enum"; + } else if (node instanceof BooleanLeaf) { + return "boolean"; + } else if (node instanceof DoubleLeaf) { + return "double"; + } else if (node instanceof FileLeaf) { + return "FileReference"; + } else if (node instanceof PathLeaf) { + return "Path"; + } else if (node instanceof IntegerLeaf) { + return "int"; + } else if (node instanceof LongLeaf) { + return "long"; + } else if (node instanceof StringLeaf) { + return "String"; + } else { + throw new IllegalStateException("Cannot determine user data type for node"); // should not occur + } + } + + /** + * @return the boxed java data type, e.g. Integer for int + */ + static String boxedDataType(CNode node) { + String rawType = userDataType(node); + + if ("int".equals(rawType)) { + return "Integer"; + } else if (rawType.toLowerCase().equals(rawType)) { + return ConfiggenUtil.capitalize(rawType); + } else { + return rawType; + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java index 299a5540098..995ef419f30 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java @@ -26,7 +26,7 @@ public class ConfiggenUtil { return className; } - private static String capitalize(String in) { + static String capitalize(String in) { StringBuilder sb = new StringBuilder(in); sb.setCharAt(0, Character.toTitleCase(in.charAt(0))); return sb.toString(); diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java new file mode 100644 index 00000000000..00498094db5 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java @@ -0,0 +1,170 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.ConfigGenerator.indentCode; +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; +import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX; + +/** + * Builds one Java class based on the given CNode tree. + * + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class JavaClassBuilder implements ClassBuilder { + public static final String INDENTATION = " "; + + private final InnerCNode root; + private final NormalizedDefinition nd; + private final String packagePrefix; + private final String javaPackage; + private final String className; + private final File destDir; + + public JavaClassBuilder(InnerCNode root, NormalizedDefinition nd, File destDir, String rawPackagePrefix) { + this.root = root; + this.nd = nd; + this.packagePrefix = (rawPackagePrefix != null) ? rawPackagePrefix : DEFAULT_PACKAGE_PREFIX; + this.javaPackage = (root.getPackage() != null) ? root.getPackage() : packagePrefix + root.getNamespace(); + this.className = createClassName(root.getName()); + this.destDir = destDir; + } + + @Override + public void createConfigClasses() { + try { + File outFile = new File(getDestPath(destDir, javaPackage), className + ".java"); + try (PrintStream out = new PrintStream(new FileOutputStream(outFile))) { + out.print(getConfigClass(className)); + } + System.err.println(outFile.getPath() + " successfully written."); + } catch (FileNotFoundException e) { + throw new CodegenRuntimeException(e); + } + } + + public String getConfigClass(String className) { + return getHeader() + "\n\n" + // + getRootClassDeclaration(root, className) + "\n\n" + // + indentCode(INDENTATION, getFrameworkCode()) + "\n\n" + // + ConfigGenerator.generateContent(INDENTATION, root, true) + "\n" + // + "}\n"; + } + + private String getHeader() { + return "/**\n" + // + " * This file is generated from a config definition file.\n" + // + " * ------------ D O N O T E D I T ! ------------\n" + // + " */\n" + // + "\n" + // + "package " + javaPackage + ";\n" + // + "\n" + // + "import java.util.*;\n" + // + "import java.nio.file.Path;\n" + // + "import edu.umd.cs.findbugs.annotations.NonNull;\n" + // + getImportFrameworkClasses(root.getNamespace()); + } + + private String getImportFrameworkClasses(String namespace) { + if (CNode.DEFAULT_NAMESPACE.equals(namespace) == false) { + return "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;"; + } else { + return ""; + } + } + + // TODO: remove the extra comment line " *" if root.getCommentBlock is empty + private String getRootClassDeclaration(InnerCNode root, String className) { + return "/**\n" + // + " * This class represents the root node of " + root.getFullName() + "\n" + // + " *\n" + // + "" + root.getCommentBlock(" *") + " */\n" + // + "public final class " + className + " extends ConfigInstance {\n" + // + "\n" + // + " public final static String CONFIG_DEF_MD5 = \"" + root.getMd5() + "\";\n" + // + " public final static String CONFIG_DEF_NAME = \"" + root.getName() + "\";\n" + // + " public final static String CONFIG_DEF_NAMESPACE = \"" + root.getNamespace() + "\";\n" + // + " public final static String CONFIG_DEF_VERSION = \"" + root.getVersion() + "\";\n" + // + " public final static String[] CONFIG_DEF_SCHEMA = {\n" + // + "" + indentCode(INDENTATION + INDENTATION, getDefSchema()) + "\n" + // + " };\n" + // + "\n" + // + " public static String getDefMd5() { return CONFIG_DEF_MD5; }\n" + // + " public static String getDefName() { return CONFIG_DEF_NAME; }\n" + // + " public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }\n" + // + " public static String getDefVersion() { return CONFIG_DEF_VERSION; }"; + } + + private String getDefSchema() { + return nd.getNormalizedContent().stream().map(l -> "\"" + l.replace("\"", "\\\"") + "\"").collect(Collectors.joining(",\n")); + } + + private String getFrameworkCode() { + return "public interface Producer extends ConfigInstance.Producer {\n" + // + " void getConfig(Builder builder);\n" + // + "}"; + } + + /** + * @param rootDir + * The root directory for the destination path. + * @param javaPackage + * The java package + * @return the destination path for the generated config file, including the + * given rootDir. + */ + private File getDestPath(File rootDir, String javaPackage) { + File dir = rootDir; + for (String subDir : javaPackage.split("\\.")) { + dir = new File(dir, subDir); + synchronized (this) { + if (!dir.isDirectory() && !dir.mkdir()) { + throw new CodegenRuntimeException("Could not create " + dir.getPath()); + } + } + } + return dir; + } + + /** + * Returns a name that can be safely used as a local variable in the generated + * config class for the given node. The name will be based on the given basis + * string, but the basis itself is not a possible return value. + * + * @param node + * The node to find a unused symbol name for. + * @param basis + * The basis for the generated symbol name. + * @return A name that is not used in the given config node. + */ + static String createUniqueSymbol(CNode node, String basis) { + Set<String> usedSymbols = Arrays.stream(node.getChildren()).map(CNode::getName).collect(Collectors.toSet()); + Random rng = new Random(); + + for (int i = 1;; i++) { + String candidate = (i < basis.length()) ? basis.substring(0, i) + : ReservedWords.INTERNAL_PREFIX + basis + rng.nextInt(Integer.MAX_VALUE); + if (usedSymbols.contains(candidate) == false) { + return candidate; + } + } + } + + public String className() { + return className; + } + + public String javaPackage() { + return javaPackage; + } +} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala deleted file mode 100644 index 4f6f310e32e..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} -import JavaClassBuilder.{Indentation, createUniqueSymbol} -import ConfigGenerator.{indentCode, nodeClass, userDataType, boxedDataType} -import com.yahoo.config.codegen.LeafCNode._ - -/** - * @author gjoranv - */ - -object BuilderGenerator { - - def getBuilder(node: InnerCNode): String = { - getDeclaration(node) + "\n" + - indentCode(Indentation, - getUninitializedScalars(node) + "\n\n" + - node.getChildren.map(getBuilderFieldDefinition).mkString("\n") + "\n\n" + - getBuilderConstructors(node, nodeClass(node)) + "\n\n" + - getOverrideMethod(node) + "\n\n" + - getBuilderSetters(node) + "\n" + - getSpecialRootBuilderCode(node) - ) + - "}" - } - - private def getDeclaration(node: InnerCNode) = { - def getInterfaces = - if (node.getParent == null) "implements ConfigInstance.Builder" - else "implements ConfigBuilder" - - "public static class Builder " + getInterfaces + " {" - } - - private def getSpecialRootBuilderCode(node: InnerCNode) = { - if (node.getParent == null) "\n" + getDispatchCode(node) + "\n" - else "" - } - - private def getDispatchCode(node: InnerCNode) = { - // Use full path to @Override, as users are free to define an inner node called 'override'. (summarymap.def does) - // The generated inner 'Override' class would otherwise be mistaken for the annotation. - """ - |@java.lang.Override - |public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { - | if (producer instanceof Producer) { - | ((Producer)producer).getConfig(this); - | return true; - | } - | return false; - |} - | - |@java.lang.Override - |public final String getDefMd5() { return CONFIG_DEF_MD5; } - |@java.lang.Override - |public final String getDefName() { return CONFIG_DEF_NAME; } - |@java.lang.Override - |public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } - """.stripMargin.trim - } - - private def getUninitializedScalars(node: InnerCNode): String = { - val scalarsWithoutDefault = { - node.getChildren.collect { - case leaf: LeafCNode if (!leaf.isArray && !leaf.isMap && leaf.getDefaultValue == null) => - "\"" + leaf.getName + "\"" - } - } - - val uninitializedList = - if (scalarsWithoutDefault.size > 0) - "Arrays.asList(\n" + indentCode(Indentation, scalarsWithoutDefault.mkString("",",\n","\n)")) - else - "" - - "private Set<String> " + InternalPrefix + "uninitialized = new HashSet<String>(" + uninitializedList + ");" - } - - private def getBuilderFieldDefinition(node: CNode): String = { - - (node match { - case array if node.isArray => - "public List<%s> %s = new ArrayList<>()".format(builderType(array), array.getName) - case map if node.isMap => - "public Map<String, %s> %s = new LinkedHashMap<>()".format(builderType(map), map.getName) - case struct: InnerCNode => - "public %s %s = new %s()".format(builderType(struct), struct.getName, builderType(struct)) - case scalar : LeafCNode => - "private " + boxedBuilderType(scalar) + " " + scalar.getName + " = null" - }) + ";" - } - - private def getBuilderSetters(node: CNode): String = { - val children: Array[CNode] = node.getChildren - - def structSetter(node: InnerCNode) = { - <code> - |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ - | {node.getName} = {InternalPrefix}builder; - | return this; - |}} - </code>.text.stripMargin.trim - } - - def innerArraySetters(node: InnerCNode) = { - <code> - |/** - | * Add the given builder to this builder's list of {nodeClass(node)} builders - | * @param {InternalPrefix}builder a builder - | * @return this builder - | */ - |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ - | {node.getName}.add({InternalPrefix}builder); - | return this; - |}} - | - |/** - | * Set the given list as this builder's list of {nodeClass(node)} builders - | * @param __builders a list of builders - | * @return this builder - | */ - |public Builder {node.getName}(List<{builderType(node)}> __builders) {{ - | {node.getName} = __builders; - | return this; - |}} - </code>.text.stripMargin.trim - } - - def leafArraySetters(node: LeafCNode) = { - val setters = - <code> - |public Builder {node.getName}({builderType(node)} {InternalPrefix}value) {{ - | {node.getName}.add({InternalPrefix}value); - | return this; - |}} - | - |public Builder {node.getName}(Collection<{builderType(node)}> {InternalPrefix}values) {{ - | {node.getName}.addAll({InternalPrefix}values); - | return this; - |}} - </code>.text.stripMargin.trim - - val privateSetter = - if (builderType(node) == "String" || builderType(node) == "FileReference") - "" - else - "\n\n" + - <code> - | - | - |private Builder {node.getName}(String {InternalPrefix}value) {{ - | return {node.getName}({builderType(node)}.valueOf({InternalPrefix}value)); - |}} - </code>.text.stripMargin.trim - - setters + privateSetter - } - - def innerMapSetters(node: CNode) = { - <code> - |public Builder {node.getName}(String {InternalPrefix}key, {builderType(node)} {InternalPrefix}value) {{ - | {node.getName}.put({InternalPrefix}key, {InternalPrefix}value); - | return this; - |}} - | - |public Builder {node.getName}(Map<String, {builderType(node)}> {InternalPrefix}values) {{ - | {node.getName}.putAll({InternalPrefix}values); - | return this; - |}} - </code>.text.stripMargin.trim - } - - def leafMapSetters(node: LeafCNode) = { - val privateSetter = - if (builderType(node) == "String" || builderType(node) == "FileReference") - "" - else - "\n\n" + - <code> - | - | - |private Builder {node.getName}(String {InternalPrefix}key, String {InternalPrefix}value) {{ - | return {node.getName}({InternalPrefix}key, {builderType(node)}.valueOf({InternalPrefix}value)); - |}} - </code>.text.stripMargin.trim - - innerMapSetters(node) + privateSetter - } - - def scalarSetters(node: LeafCNode): String = { - val name = node.getName - - val signalInitialized = - if (node.getDefaultValue == null) InternalPrefix + "uninitialized.remove(\"" + name + "\");\n" - else "" - - val stringSetter = - builderType(node) match { - case "String" => "" - case "FileReference" => "" - case _ => - """| - |private Builder %s(String %svalue) { - | return %s(%s.valueOf(%svalue)); - |}""".stripMargin.format(name, InternalPrefix, - name, boxedDataType(node), InternalPrefix) - } - - def getNullGuard = { - if (builderType(node) != boxedBuilderType(node)) - "" - else - "\n" + "if (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");" - .format(InternalPrefix) - } - - // TODO: check if 2.9.2 allows string to start with a newline - """|public Builder %s(%s %svalue) {%s - | %s = %svalue; - | %s - """.stripMargin.format(name, builderType(node), InternalPrefix, getNullGuard, - name, InternalPrefix, - signalInitialized).trim + - "\n return this;" + "\n}\n" + - stringSetter - } - - (children collect { - case innerArray: InnerCNode if innerArray.isArray => innerArraySetters(innerArray) - case innerMap: InnerCNode if innerMap.isMap => innerMapSetters(innerMap) - case leafArray: LeafCNode if leafArray.isArray => leafArraySetters(leafArray) - case leafMap: LeafCNode if leafMap.isMap => leafMapSetters(leafMap) - case struct: InnerCNode => structSetter(struct) - case scalar: LeafCNode => scalarSetters(scalar) - } ).mkString("\n\n") - } - - private def getBuilderConstructors(node: CNode, className: String): String = { - def setBuilderValueFromConfig(child: CNode) = { - val name = child.getName - val isArray = child.isArray - val isMap = child.isMap - - child match { - case fileArray: FileLeaf if isArray => name + "(" + userDataType(fileArray) + ".toValues(config." + name + "()));" - case fileMap: FileLeaf if isMap => name + "(" + userDataType(fileMap) + ".toValueMap(config." + name + "()));" - case file: FileLeaf => name + "(config." + name + "().value());" - case pathArray: PathLeaf if isArray => name + "(" + nodeClass(pathArray) + ".toFileReferences(config." + name + "));" - case pathMap: PathLeaf if isMap => name + "(" + nodeClass(pathMap) + ".toFileReferenceMap(config." + name + "));" - case path: PathLeaf => name + "(config." + name + ".getFileReference());" - case leaf: LeafCNode => name + "(config." + name + "());" - case innerArray: InnerCNode if isArray => setInnerArrayBuildersFromConfig(innerArray) - case innerMap: InnerCNode if isMap => setInnerMapBuildersFromConfig(innerMap) - case struct => name + "(new " + builderType(struct) + "(config." + name + "()));" - } - } - - def setInnerArrayBuildersFromConfig(innerArr: InnerCNode) = { - val elemName = createUniqueSymbol(node, innerArr.getName) - <code> - |for ({userDataType(innerArr)} {elemName} : config.{innerArr.getName}()) {{ - | {innerArr.getName}(new {builderType(innerArr)}({elemName})); - |}} - </code>.text.stripMargin.trim - } - - def setInnerMapBuildersFromConfig(innerMap: InnerCNode) = { - val entryName = InternalPrefix + "entry" - <code> - |for (Map.Entry<String, {userDataType(innerMap)}> {entryName} : config.{innerMap.getName}().entrySet()) {{ - | {innerMap.getName}({entryName}.getKey(), new {userDataType(innerMap)}.Builder({entryName}.getValue())); - |}} - </code>.text.stripMargin.trim - } - - <code> - |public Builder() {{ }} - | - |public Builder({className} config) {{ - |{indentCode(Indentation, node.getChildren.map(setBuilderValueFromConfig).mkString("\n"))} - |}} - </code>.text.stripMargin.trim - } - - def arrayOverride(name: String, superior: String): String = { - Indentation + name + ".addAll(" + superior + "." + name + ");" - } - - private def getOverrideMethod(node:CNode): String = { - val method = "override" - val superior = InternalPrefix + "superior" - - def callSetter(name: String): String = { - name + "(" + superior + "." + name + ");" - } - def overrideBuilderValue(child: CNode) = { - val name = child.getName - child match { - case leafArray: CNode if (child.isArray) => - conditionStatement(child) + "\n" + arrayOverride(name, superior) - case struct: InnerCNode if !(child.isArray || child.isMap) => - name + "(" + name + "." + method + "(" + superior + "." + name + "));" - case map: CNode if child.isMap => - callSetter(name) - case _ => - conditionStatement(child) + "\n" + - Indentation + callSetter(name) - } - } - - def conditionStatement(child: CNode) = { - val name = child.getName - val isArray = child.isArray - val isMap = child.isMap - child match { - case _ if isArray => "if (!" + superior + "." + name + ".isEmpty())" - case _ if isMap => "" - case scalar: LeafCNode => "if (" + superior + "." + name + " != null)" - case struct => "" - } - } - - <code> - |private Builder {method}(Builder {superior}) {{ - |{indentCode(Indentation, node.getChildren.map(overrideBuilderValue).mkString("\n"))} - | return this; - |}} - </code>.text.stripMargin.trim - } - - private def builderType(node: CNode): String = { - node match { - case inner: InnerCNode => boxedDataType(node) + ".Builder" - case file: FileLeaf => "String" - case path: PathLeaf => "FileReference" - case leafArray: LeafCNode if (node.isArray || node.isMap) => boxedDataType(node) - case _ => userDataType(node) - } - } - - private def boxedBuilderType(node: LeafCNode): String = { - node match { - case file: FileLeaf => "String" - case path: PathLeaf => "FileReference" - case _ => boxedDataType(node) - } - } - -} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala deleted file mode 100644 index 38306a03575..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - - -import com.yahoo.config.codegen.BuilderGenerator.getBuilder -import com.yahoo.config.codegen.JavaClassBuilder.Indentation -import com.yahoo.config.codegen.LeafCNode._ -import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} - -/** - * @author gjoranv - * @author tonytv - */ -// TODO: don't take indent as method param - the caller should indent -object ConfigGenerator { - - def generateContent(indent: String, node: InnerCNode, isOuter: Boolean = true): String = { - val children: Array[CNode] = node.getChildren - - def generateCodeForChildren: String = { - (children collect { - case enum: EnumLeaf => getEnumCode(enum, "") + "\n" - case inner: InnerCNode => getInnerDefinition(inner, indent) + "\n" - } ).mkString("\n") - } - - def getInnerDefinition(inner: InnerCNode, indent: String) = { - <code> - |{getClassDoc(inner, indent)} - |{getClassDeclaration(inner)} - |{generateContent(indent, inner, false)} - </code>.text.stripMargin.trim + "\n}" - } - - def getClassDeclaration(node: CNode): String = { - "public final static class " + nodeClass(node)+ " extends InnerNode { " + "\n" - } - - def getFieldDefinition(node: CNode): String = { - node.getCommentBlock("//") + "private final " + - (node match { - case _: LeafCNode if node.isArray => - "LeafNodeVector<%s, %s> %s;".format(boxedDataType(node), nodeClass(node), node.getName) - case _: InnerCNode if node.isArray => - "InnerNodeVector<%s> %s;".format(nodeClass(node), node.getName) - case _ if node.isMap => - "Map<String, %s> %s;".format(nodeClass(node), node.getName) - case _ => - "%s %s;".format(nodeClass(node), node.getName) - }) - } - - def getStaticMethods = { - if (node.isArray) getStaticMethodsForInnerArray(node) + "\n\n" - else if (node.isMap) getStaticMethodsForInnerMap(node) + "\n\n" - else "" - } - - def getContainsFieldsFlaggedWithRestart(node: CNode): String = { - if (isOuter) { - """ - |private static boolean containsFieldsFlaggedWithRestart() { - | return %b; - |} - """.stripMargin.trim.format(node.needRestart) + "\n\n" - } else "" - } - - indentCode(indent, - getBuilder(node) + "\n\n" + - children.map(getFieldDefinition).mkString("\n") + "\n\n" + - getConstructors(node) + "\n\n" + - getAccessors(children) + "\n\n" + - getGetChangesRequiringRestart(node) + "\n\n" + - getContainsFieldsFlaggedWithRestart(node) + - getStaticMethods + - generateCodeForChildren - ) - } - - private def getGetChangesRequiringRestart(node: InnerCNode): String = { - def quotedComment(node: CNode): String = { - node.getComment.replace("\n", "\\n").replace("\"", "\\\"") - } - - def getComparison(node: CNode): String = node match { - case inner: InnerCNode if inner.isArray => - <code> - | changes.compareArray(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", - | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); - </code>.text.stripMargin.trim - case inner: InnerCNode if inner.isMap => - <code> - | changes.compareMap(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", - | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); - </code>.text.stripMargin.trim - case inner: InnerCNode => - <code> - | changes.mergeChanges("{inner.getName}", this.{inner.getName}.getChangesRequiringRestart(newConfig.{inner.getName})); - </code>.text.stripMargin.trim - case node: CNode if node.isArray => - <code> - | changes.compareArray(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", - | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); - </code>.text.stripMargin.trim - case node: CNode if node.isMap => - <code> - | changes.compareMap(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", - | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); - </code>.text.stripMargin.trim - case node: CNode => - <code> - | changes.compare(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}"); - </code>.text.stripMargin.trim - } - - val comparisons = - for { - c <- node.getChildren if c.needRestart - } yield "\n " + getComparison(c) - - <code> - |private ChangesRequiringRestart getChangesRequiringRestart({nodeClass(node)} newConfig) {{ - | ChangesRequiringRestart changes = new ChangesRequiringRestart("{node.getName}");{comparisons.mkString("")} - | return changes; - |}} - </code>.text.stripMargin.trim - } - - - private def scalarDefault(scalar: LeafCNode): String = { - scalar match { - case _ if scalar.getDefaultValue == null => "" - case enumWithNullDefault: EnumLeaf if enumWithNullDefault.getDefaultValue.getValue == null => "" - case enum: EnumLeaf => nodeClass(enum) + "." + enum.getDefaultValue.getStringRepresentation - case long: LongLeaf => long.getDefaultValue.getStringRepresentation + "L" - case double: DoubleLeaf => double.getDefaultValue.getStringRepresentation + "D" - case _ => scalar.getDefaultValue.getStringRepresentation - } - } - - private def getConstructors(inner: InnerCNode) = { - - def assignFromBuilder(child: CNode) = { - val name = child.getName - val className = nodeClass(child) - val dataType = boxedDataType(child) - val isArray = child.isArray - val isMap = child.isMap - - def assignIfInitialized(leaf: LeafCNode) = { - <code> - |{name} = (builder.{name} == null) ? - | new {className}({scalarDefault(leaf)}) : new {className}(builder.{name}); - </code>.text.stripMargin.trim - } - - child match { - case fileArray: FileLeaf if isArray => - name + " = LeafNodeVector.createFileNodeVector(builder."+ name +");" - case pathArray: PathLeaf if isArray => - name + " = LeafNodeVector.createPathNodeVector(builder."+ name +");" - case leafArray: LeafCNode if isArray => - name + " = new LeafNodeVector<>(builder."+ name +", new " + className + "());" - case fileMap: LeafCNode if isMap && child.isInstanceOf[FileLeaf] => - name + " = LeafNodeMaps.asFileNodeMap(builder."+ name +");" - case pathMap: LeafCNode if isMap && child.isInstanceOf[PathLeaf] => - name + " = LeafNodeMaps.asPathNodeMap(builder."+ name +");" - case leafMap: LeafCNode if isMap => - name + " = LeafNodeMaps.asNodeMap(builder."+ name +", new " + className + "());" - case innerArray: InnerCNode if isArray => - name + " = " + className + ".createVector(builder." + name + ");" - case innerMap: InnerCNode if isMap => - name + " = " + className + ".createMap(builder." + name + ");" - case struct: InnerCNode => - name + " = new " + className + "(builder." + name + ", throwIfUninitialized);" - case leaf: LeafCNode => - assignIfInitialized(leaf) - } - } - - // TODO: merge these two constructors into one when the config library uses builders to set values from payload. - <code> - |public {nodeClass(inner)}(Builder builder) {{ - | this(builder, true); - |}} - | - |private {nodeClass(inner)}(Builder builder, boolean throwIfUninitialized) {{ - | if (throwIfUninitialized && ! builder.{InternalPrefix}uninitialized.isEmpty()) - | throw new IllegalArgumentException("The following builder parameters for " + - | "{inner.getFullName} must be initialized: " + builder.{InternalPrefix}uninitialized); - | - |{indentCode(Indentation, inner.getChildren.map(assignFromBuilder).mkString("\n"))} - |}} - </code>.text.stripMargin.trim - } - - private def getAccessors(children: Array[CNode]): String = { - - def getAccessorCode(indent: String, node: CNode): String = { - indentCode(indent, - if (node.isArray) - accessorsForArray(node) - else if (node.isMap) - accessorsForMap(node) - else - accessorForStructOrScalar(node)) - } - - def valueAccessor(node: CNode) = node match { - case leaf: LeafCNode => ".value()" - case inner => "" - } - - def listAccessor(node: CNode) = node match { - case leaf: LeafCNode => "%s.asList()".format(leaf.getName) - case inner => inner.getName - } - - def mapAccessor(node: CNode) = node match { - case leaf: LeafCNode => "LeafNodeMaps.asValueMap(%s)".format(leaf.getName) - case inner => "Collections.unmodifiableMap(%s)".format(inner.getName) - } - - def accessorsForArray(node: CNode): String = { - val name = node.getName - val fullName = node.getFullName - <code> - |/** - | * @return {fullName} - | */ - |public List<{boxedDataType(node)}> {name}() {{ - | return {listAccessor(node)}; - |}} - | - |/** - | * @param i the index of the value to return - | * @return {fullName} - | */ - |public {userDataType(node)} {name}(int i) {{ - | return {name}.get(i){valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - def accessorsForMap(node: CNode): String = { - val name = node.getName - val fullName = node.getFullName - <code> - |/** - | * @return {fullName} - | */ - |public Map<String, {boxedDataType(node)}> {name}() {{ - | return {mapAccessor(node)}; - |}} - | - |/** - | * @param key the key of the value to return - | * @return {fullName} - | */ - |public {userDataType(node)} {name}(String key) {{ - | return {name}.get(key){valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - def accessorForStructOrScalar(node: CNode): String = { - <code> - |/** - | * @return {node.getFullName} - | */ - |public {userDataType(node)} {node.getName}() {{ - | return {node.getName}{valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - val accessors = - for { - c <- children - accessor = getAccessorCode("", c) - if (accessor.length > 0) - } yield (accessor + "\n") - accessors.mkString("\n").trim - } - - private def getStaticMethodsForInnerArray(inner: InnerCNode) = { - """ - |private static InnerNodeVector<%s> createVector(List<Builder> builders) { - | List<%s> elems = new ArrayList<>(); - | for (Builder b : builders) { - | elems.add(new %s(b)); - | } - | return new InnerNodeVector<%s>(elems); - |} - """.stripMargin.format(List.fill(5)(nodeClass(inner)): _*).trim - } - - private def getStaticMethodsForInnerMap(inner: InnerCNode) = { - """ - |private static Map<String, %s> createMap(Map<String, Builder> builders) { - | Map<String, %s> ret = new LinkedHashMap<>(); - | for(String key : builders.keySet()) { - | ret.put(key, new %s(builders.get(key))); - | } - | return Collections.unmodifiableMap(ret); - |} - """.stripMargin.format(List.fill(3)(nodeClass(inner)): _*).trim - } - - private def getEnumCode(enum: EnumLeaf, indent: String): String = { - - def getEnumValues(enum: EnumLeaf): String = { - val enumValues = - for (value <- enum.getLegalValues) yield - """ public final static Enum %s = Enum.%s;""".format(value, value) - enumValues.mkString("\n") - } - - // TODO: try to rewrite to xml - val code = - """ - |%s - |public final static class %s extends EnumNode<%s> { - - | public %s(){ - | this.value = null; - | } - - | public %s(Enum enumValue) { - | super(enumValue != null); - | this.value = enumValue; - | } - - | public enum Enum {%s} - |%s - - | @Override - | protected boolean doSetValue(@NonNull String name) { - | try { - | value = Enum.valueOf(name); - | return true; - | } catch (IllegalArgumentException e) { - | } - | return false; - | } - |} - |""" - .stripMargin.format(getClassDoc(enum, indent), - nodeClass(enum), - nodeClass(enum)+".Enum", - nodeClass(enum), - nodeClass(enum), - enum.getLegalValues.mkString(", "), - getEnumValues(enum)) - - indentCode(indent, code).trim - } - - def getClassDoc(node: CNode, indent: String): String = { - val header = "/**\n" + " * This class represents " + node.getFullName - val nodeComment = node.getCommentBlock(" *") match { - case "" => "" - case s => "\n *\n" + s.stripLineEnd // TODO: strip trailing \n in CNode.getCommentBlock - } - header + nodeComment + "\n */" - } - - def indentCode(indent: String, code: String): String = { - val indentedLines = - for (s <- code.split("\n", -1)) yield - if (s.length() > 0) (indent + s) else s - indentedLines.mkString("\n") - } - - /** - * @return the name of the class that is generated by this node. - */ - def nodeClass(node: CNode): String = { - node match { - case emptyName: CNode if node.getName.length == 0 => - throw new CodegenRuntimeException("Node with empty name, under parent " + emptyName.getParent.getName) - case root: InnerCNode if root.getParent == null => ConfiggenUtil.createClassName(root.getName) - case b: BooleanLeaf => "BooleanNode" - case d: DoubleLeaf => "DoubleNode" - case f: FileLeaf => "FileNode" - case p: PathLeaf => "PathNode" - case i: IntegerLeaf => "IntegerNode" - case l: LongLeaf => "LongNode" - case r: ReferenceLeaf => "ReferenceNode" - case s: StringLeaf => "StringNode" - case _ => node.getName.capitalize - } - } - - def userDataType(node: CNode): String = { - node match { - case inner: InnerCNode => nodeClass(node) - case enum: EnumLeaf => nodeClass(enum) + ".Enum" - case b: BooleanLeaf => "boolean" - case d: DoubleLeaf => "double" - case f: FileLeaf => "FileReference" - case p: PathLeaf => "Path" - case i: IntegerLeaf => "int" - case l: LongLeaf => "long" - case s: StringLeaf => "String" - } - } - - /** - * @return the boxed java data type, e.g. Integer for int - */ - def boxedDataType(node: CNode): String = { - val rawType = userDataType(node) - - rawType match { - case "int" => "Integer" - case _ if rawType == rawType.toLowerCase => rawType.capitalize - case _ => rawType - } - } - -} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala deleted file mode 100644 index e03a6d3d04b..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import java.io.{File, FileNotFoundException, FileOutputStream, PrintStream} - -import com.yahoo.config.codegen.ConfigGenerator.indentCode -import com.yahoo.config.codegen.ConfiggenUtil.createClassName -import com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX - -import scala.collection.JavaConverters._ -import scala.util.Random -/** - * Builds one Java class based on the given CNode tree. - * - * @author gjoranv - * @author tonytv - */ -class JavaClassBuilder( - root: InnerCNode, - nd: NormalizedDefinition, - destDir: File, - rawPackagePrefix: String) - extends ClassBuilder -{ - import JavaClassBuilder._ - - val packagePrefix = if (rawPackagePrefix != null) rawPackagePrefix else DEFAULT_PACKAGE_PREFIX - val javaPackage = if (root.getPackage != null) root.getPackage else packagePrefix + root.getNamespace - val className = createClassName(root.getName) - - override def createConfigClasses() { - try { - val outFile = new File(getDestPath(destDir, javaPackage), className + ".java") - var out: PrintStream = null - try { - out = new PrintStream(new FileOutputStream(outFile)) - out.print(getConfigClass(className)) - } finally { - if (out != null) out.close() - } - System.err.println(outFile.getPath + " successfully written.") - } - catch { - case e: FileNotFoundException => { - throw new CodegenRuntimeException(e) - } - } - } - - def getConfigClass(className:String): String = { - val ret = new StringBuilder - - ret.append(getHeader).append("\n\n") - ret.append(getRootClassDeclaration(root, className)).append("\n\n") - ret.append(indentCode(Indentation, getFrameworkCode(className))).append("\n\n") - ret.append(ConfigGenerator.generateContent(Indentation, root)).append("\n") - ret.append("}\n") - - ret.toString() - } - - private def getHeader: String = { - <code> - |/** - | * This file is generated from a config definition file. - | * ------------ D O N O T E D I T ! ------------ - | */ - | - |package {javaPackage}; - | - |import java.util.*; - |import java.nio.file.Path; - |import edu.umd.cs.findbugs.annotations.NonNull; - |{getImportFrameworkClasses(root.getNamespace)} - </code>.text.stripMargin.trim - } - - private def getImportFrameworkClasses(namespace: String): String = { - if (namespace != CNode.DEFAULT_NAMESPACE) - "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;\n" - else - "" - } - - // TODO: remove the extra comment line " *" if root.getCommentBlock is empty - private def getRootClassDeclaration(root:InnerCNode, className: String): String = { - <code> - |/** - | * This class represents the root node of {root.getFullName} - | * - |{root.getCommentBlock(" *")} */ - |public final class {className} extends ConfigInstance {{ - | - | public final static String CONFIG_DEF_MD5 = "{root.getMd5}"; - | public final static String CONFIG_DEF_NAME = "{root.getName}"; - | public final static String CONFIG_DEF_NAMESPACE = "{root.getNamespace}"; - | public final static String CONFIG_DEF_VERSION = "{root.getVersion}"; - | public final static String[] CONFIG_DEF_SCHEMA = {{ - |{indentCode(Indentation * 2, getDefSchema)} - | }}; - | - | public static String getDefMd5() {{ return CONFIG_DEF_MD5; }} - | public static String getDefName() {{ return CONFIG_DEF_NAME; }} - | public static String getDefNamespace() {{ return CONFIG_DEF_NAMESPACE; }} - | public static String getDefVersion() {{ return CONFIG_DEF_VERSION; }} - </code>.text.stripMargin.trim - } - - private def getDefSchema: String = { - nd.getNormalizedContent.asScala.map { line => - "\"" + - line.replace("\"", "\\\"") + - "\"" - }.mkString(",\n") - } - - private def getFrameworkCode(className: String): String = { - getProducerBase - } - - private def getProducerBase = { - """ - |public interface Producer extends ConfigInstance.Producer { - | void getConfig(Builder builder); - |} - """.stripMargin.trim - } - - /** - * @param rootDir The root directory for the destination path. - * @param javaPackage The java package - * @return the destination path for the generated config file, including the given rootDir. - */ - private def getDestPath(rootDir: File, javaPackage: String): File = { - var dir: File = rootDir - val subDirs: Array[String] = javaPackage.split("""\.""") - for (subDir <- subDirs) { - dir = new File(dir, subDir) - this.synchronized { - if (!dir.isDirectory && !dir.mkdir) throw new CodegenRuntimeException("Could not create " + dir.getPath) - } - } - dir - } - -} - - -object JavaClassBuilder { - - val Indentation = " " - - /** - * Returns a name that can be safely used as a local variable in the generated config class - * for the given node. The name will be based on the given basis string, but the basis itself is - * not a possible return value. - * - * @param node The node to find a unused symbol name for. - * @param basis The basis for the generated symbol name. - * @return A name that is not used in the given config node. - */ - def createUniqueSymbol(node: CNode, basis: String) = { - - def getCandidate(cnt: Int) = { - if (cnt < basis.length()) - basis.substring(0, cnt) - else - ReservedWords.INTERNAL_PREFIX + basis + Random.nextInt().abs - } - - def getUsedSymbols: Set[String] = { - (node.getChildren map (child => child.getName)).toSet - } - - // TODO: refactoring potential - val usedSymbols = getUsedSymbols - var count = 1 - var candidate = getCandidate(count) - while (usedSymbols contains(candidate)) { - count += 1 - candidate = getCandidate(count) - } - candidate - } - -} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java new file mode 100644 index 00000000000..744f8c9b1a2 --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java @@ -0,0 +1,116 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.FileSystems; +import java.nio.file.Files; + +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; +import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + * @author ollivir + */ +public class JavaClassBuilderTest { + private static final String TEST_DIR = "target/test-classes/"; + private static final String DEF_NAME = TEST_DIR + "allfeatures.def"; + private static final String REFERENCE_NAME = TEST_DIR + "allfeatures.reference"; + + @Ignore + @Test + public void visual_inspection_of_generated_class() { + final String testDefinition = "version=1\n" + // + "namespace=test\n" + // + "p path\n" + // + "pathArr[] path\n" + // + "f file\n" + // + "fileArr[] file\n" + // + "i int default=0\n" + // + "# A long value\n" + // + "l long default=0\n" + // + "s string default=\"\"\n" + // + "b bool\n" + // + "# An enum value\n" + // + "e enum {A, B, C}\n" + // + "intArr[] int\n" + // + "boolArr[] bool\n" + // + "enumArr[] enum {FOO, BAR}\n" + // + "intMap{} int\n" + // + "# A struct\n" + // + "# with multi-line\n" + // + "# comment and \"quotes\".\n" + // + "myStruct.i int\n" + // + "myStruct.s string\n" + // + "# An inner array\n" + // + "myArr[].i int\n" + // + "myArr[].newStruct.s string\n" + // + "myArr[].newStruct.b bool\n" + // + "myArr[].intArr[] int\n" + // + "# An inner map\n" + // + "myMap{}.i int\n" + // + "myMap{}.newStruct.s string\n" + // + "myMap{}.newStruct.b bool\n" + // + "myMap{}.intArr[] int\n" + // + "intMap{} int\n"; + + DefParser parser = new DefParser("test", new StringReader(testDefinition)); + InnerCNode root = parser.getTree(); + JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null); + String configClass = builder.getConfigClass("TestConfig"); + System.out.print(configClass); + } + + @Test + public void testCreateUniqueSymbol() { + final String testDefinition = "version=1\n" + // + "namespace=test\n" + // + "m int\n" + // + "n int\n"; + InnerCNode root = new DefParser("test", new StringReader(testDefinition)).getTree(); + + assertThat(createUniqueSymbol(root, "foo"), is("f")); + assertThat(createUniqueSymbol(root, "name"), is("na")); + assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")); + + // The basis string is not a legal return value, even if unique, to avoid + // multiple symbols with the same name if the same basis string is given twice. + assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my")); + } + + @Test + public void testCreateClassName() { + assertThat(createClassName("simple"), is("SimpleConfig")); + assertThat(createClassName("a"), is("AConfig")); + assertThat(createClassName("a-b-c"), is("ABCConfig")); + assertThat(createClassName("a-1-2b"), is("A12bConfig")); + assertThat(createClassName("my-app"), is("MyAppConfig")); + assertThat(createClassName("MyApp"), is("MyAppConfig")); + } + + @Test(expected = CodegenRuntimeException.class) + public void testIllegalClassName() { + createClassName("+illegal"); + } + + @Test + public void verify_generated_class_against_reference() throws IOException { + final String testDefinition = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(DEF_NAME))); + final String referenceClass = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(REFERENCE_NAME))) + "\n"; + + DefParser parser = new DefParser("allfeatures", new StringReader(testDefinition)); + InnerCNode root = parser.getTree(); + JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null); + String configClass = builder.getConfigClass("AllfeaturesConfig"); + + assertEquals(referenceClass, configClass); + } +} diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference new file mode 100644 index 00000000000..ebc21e8255c --- /dev/null +++ b/configgen/src/test/resources/allfeatures.reference @@ -0,0 +1,1983 @@ +/** + * This file is generated from a config definition file. + * ------------ D O N O T E D I T ! ------------ + */ + +package com.yahoo.configgen; + +import java.util.*; +import java.nio.file.Path; +import edu.umd.cs.findbugs.annotations.NonNull; +import com.yahoo.config.*; + +/** + * This class represents the root node of allfeatures + * + * Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * This def file should test most aspects of def files that makes a difference + * for the generated config classes. The goal is to trigger all blocks of + * code in the code generators. This includes: + * + * - Use all legal special characters in the def file name, to ensure that those + * that needs to be replaced in type names are actually replaced. + * - Use the same enum type twice to verify that we dont declare or define it + * twice. + * - Use the same struct type twice for the same reason. + * - Include arrays of primitives and structs. + * - Include enum primitives and array of enums. Arrays of enums must be handled + * specially by the C++ code. + * - Include enums both with and without default values. + * - Include primitive string, numbers & doubles both with and without default + * values. + * - Have an array within a struct, to verify that we correctly recurse. + * - Reuse type name further within to ensure that this works. + */ +public final class AllfeaturesConfig extends ConfigInstance { + + public final static String CONFIG_DEF_MD5 = "eb2d24dbbcf054b21be729e2cfaafd93"; + public final static String CONFIG_DEF_NAME = "allfeatures"; + public final static String CONFIG_DEF_NAMESPACE = "configgen"; + public final static String CONFIG_DEF_VERSION = ""; + public final static String[] CONFIG_DEF_SCHEMA = { + "namespace=configgen", + "boolVal bool", + "bool_with_def bool default=false", + "intVal int", + "intWithDef int default=-545", + "longVal long", + "longWithDef long default=1234567890123", + "doubleVal double", + "double_with_def double default=-6.43", + "stringVal string", + "stringwithdef string default=\"foobar#notacomment\"", + "enumVal enum { FOO, BAR, FOOBAR }", + "enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2", + "refVal reference", + "refwithdef reference default=\":parent:\"", + "fileVal file", + "pathVal path", + "boolarr[] bool", + "intarr[] int", + "longarr[] long", + "doublearr[] double", + "stringarr[] string", + "enumarr[] enum { ARRAY, VALUES }", + "refarr[] reference", + "filearr[] file", + "pathArr[] path", + "intMap{} int", + "pathMap{} file", + "basic_struct.foo string default=\"foo\"", + "basic_struct.bar int default=0", + "struct_of_struct.inner0.name string default=\"inner0\"", + "struct_of_struct.inner0.index int default=0", + "struct_of_struct.inner1.name string default=\"inner1\"", + "struct_of_struct.inner1.index int default=1", + "myArray[].intVal int default=14", + "myArray[].stringVal[] string", + "myArray[].enumVal enum { INNER, ENUM, TYPE } default=TYPE", + "myArray[].refVal reference", + "myArray[].anotherArray[].foo int default=-4", + "myMap{}.intVal int default=15", + "myMap{}.stringVal[] string", + "myMap{}.enumVal enum { INNER, ENUM, TYPE } default=ENUM", + "myMap{}.refVal reference", + "myMap{}.anotherArray[].foo int default=-5" + }; + + public static String getDefMd5() { return CONFIG_DEF_MD5; } + public static String getDefName() { return CONFIG_DEF_NAME; } + public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + public static String getDefVersion() { return CONFIG_DEF_VERSION; } + + public interface Producer extends ConfigInstance.Producer { + void getConfig(Builder builder); + } + + public static class Builder implements ConfigInstance.Builder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "boolVal", + "intVal", + "longVal", + "doubleVal", + "stringVal", + "enumVal", + "refVal", + "fileVal", + "pathVal" + )); + + private Boolean boolVal = null; + private Boolean bool_with_def = null; + private Integer intVal = null; + private Integer intWithDef = null; + private Long longVal = null; + private Long longWithDef = null; + private Double doubleVal = null; + private Double double_with_def = null; + private String stringVal = null; + private String stringwithdef = null; + private EnumVal.Enum enumVal = null; + private Enumwithdef.Enum enumwithdef = null; + private String refVal = null; + private String refwithdef = null; + private String fileVal = null; + private FileReference pathVal = null; + public List<Boolean> boolarr = new ArrayList<>(); + public List<Integer> intarr = new ArrayList<>(); + public List<Long> longarr = new ArrayList<>(); + public List<Double> doublearr = new ArrayList<>(); + public List<String> stringarr = new ArrayList<>(); + public List<Enumarr.Enum> enumarr = new ArrayList<>(); + public List<String> refarr = new ArrayList<>(); + public List<String> filearr = new ArrayList<>(); + public List<FileReference> pathArr = new ArrayList<>(); + public Map<String, Integer> intMap = new LinkedHashMap<>(); + public Map<String, String> pathMap = new LinkedHashMap<>(); + public Basic_struct.Builder basic_struct = new Basic_struct.Builder(); + public Struct_of_struct.Builder struct_of_struct = new Struct_of_struct.Builder(); + public List<MyArray.Builder> myArray = new ArrayList<>(); + public Map<String, MyMap.Builder> myMap = new LinkedHashMap<>(); + + public Builder() { } + + public Builder(AllfeaturesConfig config) { + boolVal(config.boolVal()); + bool_with_def(config.bool_with_def()); + intVal(config.intVal()); + intWithDef(config.intWithDef()); + longVal(config.longVal()); + longWithDef(config.longWithDef()); + doubleVal(config.doubleVal()); + double_with_def(config.double_with_def()); + stringVal(config.stringVal()); + stringwithdef(config.stringwithdef()); + enumVal(config.enumVal()); + enumwithdef(config.enumwithdef()); + refVal(config.refVal()); + refwithdef(config.refwithdef()); + fileVal(config.fileVal().value()); + pathVal(config.pathVal.getFileReference()); + boolarr(config.boolarr()); + intarr(config.intarr()); + longarr(config.longarr()); + doublearr(config.doublearr()); + stringarr(config.stringarr()); + enumarr(config.enumarr()); + refarr(config.refarr()); + filearr(FileReference.toValues(config.filearr())); + pathArr(PathNode.toFileReferences(config.pathArr)); + intMap(config.intMap()); + pathMap(FileReference.toValueMap(config.pathMap())); + basic_struct(new Basic_struct.Builder(config.basic_struct())); + struct_of_struct(new Struct_of_struct.Builder(config.struct_of_struct())); + for (MyArray m : config.myArray()) { + myArray(new MyArray.Builder(m)); + } + for (Map.Entry<String, MyMap> __entry : config.myMap().entrySet()) { + myMap(__entry.getKey(), new MyMap.Builder(__entry.getValue())); + } + } + + private Builder override(Builder __superior) { + if (__superior.boolVal != null) + boolVal(__superior.boolVal); + if (__superior.bool_with_def != null) + bool_with_def(__superior.bool_with_def); + if (__superior.intVal != null) + intVal(__superior.intVal); + if (__superior.intWithDef != null) + intWithDef(__superior.intWithDef); + if (__superior.longVal != null) + longVal(__superior.longVal); + if (__superior.longWithDef != null) + longWithDef(__superior.longWithDef); + if (__superior.doubleVal != null) + doubleVal(__superior.doubleVal); + if (__superior.double_with_def != null) + double_with_def(__superior.double_with_def); + if (__superior.stringVal != null) + stringVal(__superior.stringVal); + if (__superior.stringwithdef != null) + stringwithdef(__superior.stringwithdef); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.enumwithdef != null) + enumwithdef(__superior.enumwithdef); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (__superior.refwithdef != null) + refwithdef(__superior.refwithdef); + if (__superior.fileVal != null) + fileVal(__superior.fileVal); + if (__superior.pathVal != null) + pathVal(__superior.pathVal); + if (!__superior.boolarr.isEmpty()) + boolarr.addAll(__superior.boolarr); + if (!__superior.intarr.isEmpty()) + intarr.addAll(__superior.intarr); + if (!__superior.longarr.isEmpty()) + longarr.addAll(__superior.longarr); + if (!__superior.doublearr.isEmpty()) + doublearr.addAll(__superior.doublearr); + if (!__superior.stringarr.isEmpty()) + stringarr.addAll(__superior.stringarr); + if (!__superior.enumarr.isEmpty()) + enumarr.addAll(__superior.enumarr); + if (!__superior.refarr.isEmpty()) + refarr.addAll(__superior.refarr); + if (!__superior.filearr.isEmpty()) + filearr.addAll(__superior.filearr); + if (!__superior.pathArr.isEmpty()) + pathArr.addAll(__superior.pathArr); + intMap(__superior.intMap); + pathMap(__superior.pathMap); + basic_struct(basic_struct.override(__superior.basic_struct)); + struct_of_struct(struct_of_struct.override(__superior.struct_of_struct)); + if (!__superior.myArray.isEmpty()) + myArray.addAll(__superior.myArray); + myMap(__superior.myMap); + return this; + } + + public Builder boolVal(boolean __value) { + boolVal = __value; + __uninitialized.remove("boolVal"); + return this; + } + + private Builder boolVal(String __value) { + return boolVal(Boolean.valueOf(__value)); + } + + public Builder bool_with_def(boolean __value) { + bool_with_def = __value; + return this; + } + + private Builder bool_with_def(String __value) { + return bool_with_def(Boolean.valueOf(__value)); + } + + public Builder intVal(int __value) { + intVal = __value; + __uninitialized.remove("intVal"); + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder intWithDef(int __value) { + intWithDef = __value; + return this; + } + + private Builder intWithDef(String __value) { + return intWithDef(Integer.valueOf(__value)); + } + + public Builder longVal(long __value) { + longVal = __value; + __uninitialized.remove("longVal"); + return this; + } + + private Builder longVal(String __value) { + return longVal(Long.valueOf(__value)); + } + + public Builder longWithDef(long __value) { + longWithDef = __value; + return this; + } + + private Builder longWithDef(String __value) { + return longWithDef(Long.valueOf(__value)); + } + + public Builder doubleVal(double __value) { + doubleVal = __value; + __uninitialized.remove("doubleVal"); + return this; + } + + private Builder doubleVal(String __value) { + return doubleVal(Double.valueOf(__value)); + } + + public Builder double_with_def(double __value) { + double_with_def = __value; + return this; + } + + private Builder double_with_def(String __value) { + return double_with_def(Double.valueOf(__value)); + } + + public Builder stringVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + stringVal = __value; + __uninitialized.remove("stringVal"); + return this; + } + + + public Builder stringwithdef(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + stringwithdef = __value; + return this; + } + + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + __uninitialized.remove("enumVal"); + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder enumwithdef(Enumwithdef.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumwithdef = __value; + return this; + } + + private Builder enumwithdef(String __value) { + return enumwithdef(Enumwithdef.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + public Builder refwithdef(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refwithdef = __value; + return this; + } + + + public Builder fileVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + fileVal = __value; + __uninitialized.remove("fileVal"); + return this; + } + + + public Builder pathVal(FileReference __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + pathVal = __value; + __uninitialized.remove("pathVal"); + return this; + } + + + public Builder boolarr(Boolean __value) { + boolarr.add(__value); + return this; + } + + public Builder boolarr(Collection<Boolean> __values) { + boolarr.addAll(__values); + return this; + } + + private Builder boolarr(String __value) { + return boolarr(Boolean.valueOf(__value)); + } + + public Builder intarr(Integer __value) { + intarr.add(__value); + return this; + } + + public Builder intarr(Collection<Integer> __values) { + intarr.addAll(__values); + return this; + } + + private Builder intarr(String __value) { + return intarr(Integer.valueOf(__value)); + } + + public Builder longarr(Long __value) { + longarr.add(__value); + return this; + } + + public Builder longarr(Collection<Long> __values) { + longarr.addAll(__values); + return this; + } + + private Builder longarr(String __value) { + return longarr(Long.valueOf(__value)); + } + + public Builder doublearr(Double __value) { + doublearr.add(__value); + return this; + } + + public Builder doublearr(Collection<Double> __values) { + doublearr.addAll(__values); + return this; + } + + private Builder doublearr(String __value) { + return doublearr(Double.valueOf(__value)); + } + + public Builder stringarr(String __value) { + stringarr.add(__value); + return this; + } + + public Builder stringarr(Collection<String> __values) { + stringarr.addAll(__values); + return this; + } + + public Builder enumarr(Enumarr.Enum __value) { + enumarr.add(__value); + return this; + } + + public Builder enumarr(Collection<Enumarr.Enum> __values) { + enumarr.addAll(__values); + return this; + } + + private Builder enumarr(String __value) { + return enumarr(Enumarr.Enum.valueOf(__value)); + } + + public Builder refarr(String __value) { + refarr.add(__value); + return this; + } + + public Builder refarr(Collection<String> __values) { + refarr.addAll(__values); + return this; + } + + public Builder filearr(String __value) { + filearr.add(__value); + return this; + } + + public Builder filearr(Collection<String> __values) { + filearr.addAll(__values); + return this; + } + + public Builder pathArr(FileReference __value) { + pathArr.add(__value); + return this; + } + + public Builder pathArr(Collection<FileReference> __values) { + pathArr.addAll(__values); + return this; + } + + public Builder intMap(String __key, Integer __value) { + intMap.put(__key, __value); + return this; + } + + public Builder intMap(Map<String, Integer> __values) { + intMap.putAll(__values); + return this; + } + + private Builder intMap(String __key, String __value) { + return intMap(__key, Integer.valueOf(__value)); + } + + public Builder pathMap(String __key, String __value) { + pathMap.put(__key, __value); + return this; + } + + public Builder pathMap(Map<String, String> __values) { + pathMap.putAll(__values); + return this; + } + + public Builder basic_struct(Basic_struct.Builder __builder) { + basic_struct = __builder; + return this; + } + + public Builder struct_of_struct(Struct_of_struct.Builder __builder) { + struct_of_struct = __builder; + return this; + } + + /** + * Add the given builder to this builder's list of MyArray builders + * @param __builder a builder + * @return this builder + */ + public Builder myArray(MyArray.Builder __builder) { + myArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of MyArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder myArray(List<MyArray.Builder> __builders) { + myArray = __builders; + return this; + } + + public Builder myMap(String __key, MyMap.Builder __value) { + myMap.put(__key, __value); + return this; + } + + public Builder myMap(Map<String, MyMap.Builder> __values) { + myMap.putAll(__values); + return this; + } + + @java.lang.Override + public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { + if (producer instanceof Producer) { + ((Producer)producer).getConfig(this); + return true; + } + return false; + } + + @java.lang.Override + public final String getDefMd5() { return CONFIG_DEF_MD5; } + @java.lang.Override + public final String getDefName() { return CONFIG_DEF_NAME; } + @java.lang.Override + public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + } + + // Some random bool without a default value. These comments exist to check + // that comment parsing works.e + private final BooleanNode boolVal; + // A bool with a default value set. + private final BooleanNode bool_with_def; + private final IntegerNode intVal; + private final IntegerNode intWithDef; + private final LongNode longVal; + private final LongNode longWithDef; + private final DoubleNode doubleVal; + private final DoubleNode double_with_def; + // Another comment + private final StringNode stringVal; + private final StringNode stringwithdef; + private final EnumVal enumVal; + private final Enumwithdef enumwithdef; + private final ReferenceNode refVal; + private final ReferenceNode refwithdef; + private final FileNode fileVal; + private final PathNode pathVal; + private final LeafNodeVector<Boolean, BooleanNode> boolarr; + private final LeafNodeVector<Integer, IntegerNode> intarr; + private final LeafNodeVector<Long, LongNode> longarr; + private final LeafNodeVector<Double, DoubleNode> doublearr; + private final LeafNodeVector<String, StringNode> stringarr; + private final LeafNodeVector<Enumarr.Enum, Enumarr> enumarr; + private final LeafNodeVector<String, ReferenceNode> refarr; + private final LeafNodeVector<FileReference, FileNode> filearr; + private final LeafNodeVector<Path, PathNode> pathArr; + private final Map<String, IntegerNode> intMap; + private final Map<String, FileNode> pathMap; + private final Basic_struct basic_struct; + private final Struct_of_struct struct_of_struct; + private final InnerNodeVector<MyArray> myArray; + private final Map<String, MyMap> myMap; + + public AllfeaturesConfig(Builder builder) { + this(builder, true); + } + + private AllfeaturesConfig(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures must be initialized: " + builder.__uninitialized); + + boolVal = (builder.boolVal == null) ? + new BooleanNode() : new BooleanNode(builder.boolVal); + bool_with_def = (builder.bool_with_def == null) ? + new BooleanNode(false) : new BooleanNode(builder.bool_with_def); + intVal = (builder.intVal == null) ? + new IntegerNode() : new IntegerNode(builder.intVal); + intWithDef = (builder.intWithDef == null) ? + new IntegerNode(-545) : new IntegerNode(builder.intWithDef); + longVal = (builder.longVal == null) ? + new LongNode() : new LongNode(builder.longVal); + longWithDef = (builder.longWithDef == null) ? + new LongNode(1234567890123L) : new LongNode(builder.longWithDef); + doubleVal = (builder.doubleVal == null) ? + new DoubleNode() : new DoubleNode(builder.doubleVal); + double_with_def = (builder.double_with_def == null) ? + new DoubleNode(-6.43D) : new DoubleNode(builder.double_with_def); + stringVal = (builder.stringVal == null) ? + new StringNode() : new StringNode(builder.stringVal); + stringwithdef = (builder.stringwithdef == null) ? + new StringNode("foobar#notacomment") : new StringNode(builder.stringwithdef); + enumVal = (builder.enumVal == null) ? + new EnumVal() : new EnumVal(builder.enumVal); + enumwithdef = (builder.enumwithdef == null) ? + new Enumwithdef(Enumwithdef.BAR2) : new Enumwithdef(builder.enumwithdef); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + refwithdef = (builder.refwithdef == null) ? + new ReferenceNode(":parent:") : new ReferenceNode(builder.refwithdef); + fileVal = (builder.fileVal == null) ? + new FileNode() : new FileNode(builder.fileVal); + pathVal = (builder.pathVal == null) ? + new PathNode() : new PathNode(builder.pathVal); + boolarr = new LeafNodeVector<>(builder.boolarr, new BooleanNode()); + intarr = new LeafNodeVector<>(builder.intarr, new IntegerNode()); + longarr = new LeafNodeVector<>(builder.longarr, new LongNode()); + doublearr = new LeafNodeVector<>(builder.doublearr, new DoubleNode()); + stringarr = new LeafNodeVector<>(builder.stringarr, new StringNode()); + enumarr = new LeafNodeVector<>(builder.enumarr, new Enumarr()); + refarr = new LeafNodeVector<>(builder.refarr, new ReferenceNode()); + filearr = LeafNodeVector.createFileNodeVector(builder.filearr); + pathArr = LeafNodeVector.createPathNodeVector(builder.pathArr); + intMap = LeafNodeMaps.asNodeMap(builder.intMap, new IntegerNode()); + pathMap = LeafNodeMaps.asFileNodeMap(builder.pathMap); + basic_struct = new Basic_struct(builder.basic_struct, throwIfUninitialized); + struct_of_struct = new Struct_of_struct(builder.struct_of_struct, throwIfUninitialized); + myArray = MyArray.createVector(builder.myArray); + myMap = MyMap.createMap(builder.myMap); + } + + /** + * @return allfeatures.boolVal + */ + public boolean boolVal() { + return boolVal.value(); + } + + /** + * @return allfeatures.bool_with_def + */ + public boolean bool_with_def() { + return bool_with_def.value(); + } + + /** + * @return allfeatures.intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.intWithDef + */ + public int intWithDef() { + return intWithDef.value(); + } + + /** + * @return allfeatures.longVal + */ + public long longVal() { + return longVal.value(); + } + + /** + * @return allfeatures.longWithDef + */ + public long longWithDef() { + return longWithDef.value(); + } + + /** + * @return allfeatures.doubleVal + */ + public double doubleVal() { + return doubleVal.value(); + } + + /** + * @return allfeatures.double_with_def + */ + public double double_with_def() { + return double_with_def.value(); + } + + /** + * @return allfeatures.stringVal + */ + public String stringVal() { + return stringVal.value(); + } + + /** + * @return allfeatures.stringwithdef + */ + public String stringwithdef() { + return stringwithdef.value(); + } + + /** + * @return allfeatures.enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.enumwithdef + */ + public Enumwithdef.Enum enumwithdef() { + return enumwithdef.value(); + } + + /** + * @return allfeatures.refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.refwithdef + */ + public String refwithdef() { + return refwithdef.value(); + } + + /** + * @return allfeatures.fileVal + */ + public FileReference fileVal() { + return fileVal.value(); + } + + /** + * @return allfeatures.pathVal + */ + public Path pathVal() { + return pathVal.value(); + } + + /** + * @return allfeatures.boolarr[] + */ + public List<Boolean> boolarr() { + return boolarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.boolarr[] + */ + public boolean boolarr(int i) { + return boolarr.get(i).value(); + } + + /** + * @return allfeatures.intarr[] + */ + public List<Integer> intarr() { + return intarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.intarr[] + */ + public int intarr(int i) { + return intarr.get(i).value(); + } + + /** + * @return allfeatures.longarr[] + */ + public List<Long> longarr() { + return longarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.longarr[] + */ + public long longarr(int i) { + return longarr.get(i).value(); + } + + /** + * @return allfeatures.doublearr[] + */ + public List<Double> doublearr() { + return doublearr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.doublearr[] + */ + public double doublearr(int i) { + return doublearr.get(i).value(); + } + + /** + * @return allfeatures.stringarr[] + */ + public List<String> stringarr() { + return stringarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.stringarr[] + */ + public String stringarr(int i) { + return stringarr.get(i).value(); + } + + /** + * @return allfeatures.enumarr[] + */ + public List<Enumarr.Enum> enumarr() { + return enumarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.enumarr[] + */ + public Enumarr.Enum enumarr(int i) { + return enumarr.get(i).value(); + } + + /** + * @return allfeatures.refarr[] + */ + public List<String> refarr() { + return refarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.refarr[] + */ + public String refarr(int i) { + return refarr.get(i).value(); + } + + /** + * @return allfeatures.filearr[] + */ + public List<FileReference> filearr() { + return filearr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.filearr[] + */ + public FileReference filearr(int i) { + return filearr.get(i).value(); + } + + /** + * @return allfeatures.pathArr[] + */ + public List<Path> pathArr() { + return pathArr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.pathArr[] + */ + public Path pathArr(int i) { + return pathArr.get(i).value(); + } + + /** + * @return allfeatures.intMap{} + */ + public Map<String, Integer> intMap() { + return LeafNodeMaps.asValueMap(intMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.intMap{} + */ + public int intMap(String key) { + return intMap.get(key).value(); + } + + /** + * @return allfeatures.pathMap{} + */ + public Map<String, FileReference> pathMap() { + return LeafNodeMaps.asValueMap(pathMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.pathMap{} + */ + public FileReference pathMap(String key) { + return pathMap.get(key).value(); + } + + /** + * @return allfeatures.basic_struct + */ + public Basic_struct basic_struct() { + return basic_struct; + } + + /** + * @return allfeatures.struct_of_struct + */ + public Struct_of_struct struct_of_struct() { + return struct_of_struct; + } + + /** + * @return allfeatures.myArray[] + */ + public List<MyArray> myArray() { + return myArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[] + */ + public MyArray myArray(int i) { + return myArray.get(i); + } + + /** + * @return allfeatures.myMap{} + */ + public Map<String, MyMap> myMap() { + return Collections.unmodifiableMap(myMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.myMap{} + */ + public MyMap myMap(String key) { + return myMap.get(key); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AllfeaturesConfig newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("allfeatures"); + return changes; + } + + private static boolean containsFieldsFlaggedWithRestart() { + return false; + } + + /** + * This class represents allfeatures.enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {FOO, BAR, FOOBAR} + public final static Enum FOO = Enum.FOO; + public final static Enum BAR = Enum.BAR; + public final static Enum FOOBAR = Enum.FOOBAR; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.enumwithdef + */ + public final static class Enumwithdef extends EnumNode<Enumwithdef.Enum> { + + public Enumwithdef(){ + this.value = null; + } + + public Enumwithdef(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {FOO2, BAR2, FOOBAR2} + public final static Enum FOO2 = Enum.FOO2; + public final static Enum BAR2 = Enum.BAR2; + public final static Enum FOOBAR2 = Enum.FOOBAR2; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.enumarr[] + */ + public final static class Enumarr extends EnumNode<Enumarr.Enum> { + + public Enumarr(){ + this.value = null; + } + + public Enumarr(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {ARRAY, VALUES} + public final static Enum ARRAY = Enum.ARRAY; + public final static Enum VALUES = Enum.VALUES; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.basic_struct + */ + public final static class Basic_struct extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String foo = null; + private Integer bar = null; + + public Builder() { } + + public Builder(Basic_struct config) { + foo(config.foo()); + bar(config.bar()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + if (__superior.bar != null) + bar(__superior.bar); + return this; + } + + public Builder foo(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + foo = __value; + return this; + } + + + public Builder bar(int __value) { + bar = __value; + return this; + } + + private Builder bar(String __value) { + return bar(Integer.valueOf(__value)); + } + } + + // A basic struct + private final StringNode foo; + private final IntegerNode bar; + + public Basic_struct(Builder builder) { + this(builder, true); + } + + private Basic_struct(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.basic_struct must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new StringNode("foo") : new StringNode(builder.foo); + bar = (builder.bar == null) ? + new IntegerNode(0) : new IntegerNode(builder.bar); + } + + /** + * @return allfeatures.basic_struct.foo + */ + public String foo() { + return foo.value(); + } + + /** + * @return allfeatures.basic_struct.bar + */ + public int bar() { + return bar.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Basic_struct newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("basic_struct"); + return changes; + } + } + + /** + * This class represents allfeatures.struct_of_struct + */ + public final static class Struct_of_struct extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + public Inner0.Builder inner0 = new Inner0.Builder(); + public Inner1.Builder inner1 = new Inner1.Builder(); + + public Builder() { } + + public Builder(Struct_of_struct config) { + inner0(new Inner0.Builder(config.inner0())); + inner1(new Inner1.Builder(config.inner1())); + } + + private Builder override(Builder __superior) { + inner0(inner0.override(__superior.inner0)); + inner1(inner1.override(__superior.inner1)); + return this; + } + + public Builder inner0(Inner0.Builder __builder) { + inner0 = __builder; + return this; + } + + public Builder inner1(Inner1.Builder __builder) { + inner1 = __builder; + return this; + } + } + + private final Inner0 inner0; + private final Inner1 inner1; + + public Struct_of_struct(Builder builder) { + this(builder, true); + } + + private Struct_of_struct(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct must be initialized: " + builder.__uninitialized); + + inner0 = new Inner0(builder.inner0, throwIfUninitialized); + inner1 = new Inner1(builder.inner1, throwIfUninitialized); + } + + /** + * @return allfeatures.struct_of_struct.inner0 + */ + public Inner0 inner0() { + return inner0; + } + + /** + * @return allfeatures.struct_of_struct.inner1 + */ + public Inner1 inner1() { + return inner1; + } + + private ChangesRequiringRestart getChangesRequiringRestart(Struct_of_struct newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("struct_of_struct"); + return changes; + } + + /** + * This class represents allfeatures.struct_of_struct.inner0 + */ + public final static class Inner0 extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String name = null; + private Integer index = null; + + public Builder() { } + + public Builder(Inner0 config) { + name(config.name()); + index(config.index()); + } + + private Builder override(Builder __superior) { + if (__superior.name != null) + name(__superior.name); + if (__superior.index != null) + index(__superior.index); + return this; + } + + public Builder name(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + name = __value; + return this; + } + + + public Builder index(int __value) { + index = __value; + return this; + } + + private Builder index(String __value) { + return index(Integer.valueOf(__value)); + } + } + + // A struct of struct + private final StringNode name; + private final IntegerNode index; + + public Inner0(Builder builder) { + this(builder, true); + } + + private Inner0(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct.inner0 must be initialized: " + builder.__uninitialized); + + name = (builder.name == null) ? + new StringNode("inner0") : new StringNode(builder.name); + index = (builder.index == null) ? + new IntegerNode(0) : new IntegerNode(builder.index); + } + + /** + * @return allfeatures.struct_of_struct.inner0.name + */ + public String name() { + return name.value(); + } + + /** + * @return allfeatures.struct_of_struct.inner0.index + */ + public int index() { + return index.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Inner0 newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("inner0"); + return changes; + } + } + + /** + * This class represents allfeatures.struct_of_struct.inner1 + */ + public final static class Inner1 extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String name = null; + private Integer index = null; + + public Builder() { } + + public Builder(Inner1 config) { + name(config.name()); + index(config.index()); + } + + private Builder override(Builder __superior) { + if (__superior.name != null) + name(__superior.name); + if (__superior.index != null) + index(__superior.index); + return this; + } + + public Builder name(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + name = __value; + return this; + } + + + public Builder index(int __value) { + index = __value; + return this; + } + + private Builder index(String __value) { + return index(Integer.valueOf(__value)); + } + } + + private final StringNode name; + private final IntegerNode index; + + public Inner1(Builder builder) { + this(builder, true); + } + + private Inner1(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct.inner1 must be initialized: " + builder.__uninitialized); + + name = (builder.name == null) ? + new StringNode("inner1") : new StringNode(builder.name); + index = (builder.index == null) ? + new IntegerNode(1) : new IntegerNode(builder.index); + } + + /** + * @return allfeatures.struct_of_struct.inner1.name + */ + public String name() { + return name.value(); + } + + /** + * @return allfeatures.struct_of_struct.inner1.index + */ + public int index() { + return index.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Inner1 newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("inner1"); + return changes; + } + } + } + + /** + * This class represents allfeatures.myArray[] + */ + public final static class MyArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "refVal" + )); + + private Integer intVal = null; + public List<String> stringVal = new ArrayList<>(); + private EnumVal.Enum enumVal = null; + private String refVal = null; + public List<AnotherArray.Builder> anotherArray = new ArrayList<>(); + + public Builder() { } + + public Builder(MyArray config) { + intVal(config.intVal()); + stringVal(config.stringVal()); + enumVal(config.enumVal()); + refVal(config.refVal()); + for (AnotherArray a : config.anotherArray()) { + anotherArray(new AnotherArray.Builder(a)); + } + } + + private Builder override(Builder __superior) { + if (__superior.intVal != null) + intVal(__superior.intVal); + if (!__superior.stringVal.isEmpty()) + stringVal.addAll(__superior.stringVal); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (!__superior.anotherArray.isEmpty()) + anotherArray.addAll(__superior.anotherArray); + return this; + } + + public Builder intVal(int __value) { + intVal = __value; + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder stringVal(String __value) { + stringVal.add(__value); + return this; + } + + public Builder stringVal(Collection<String> __values) { + stringVal.addAll(__values); + return this; + } + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + /** + * Add the given builder to this builder's list of AnotherArray builders + * @param __builder a builder + * @return this builder + */ + public Builder anotherArray(AnotherArray.Builder __builder) { + anotherArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of AnotherArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder anotherArray(List<AnotherArray.Builder> __builders) { + anotherArray = __builders; + return this; + } + } + + private final IntegerNode intVal; + private final LeafNodeVector<String, StringNode> stringVal; + private final EnumVal enumVal; + private final ReferenceNode refVal; + private final InnerNodeVector<AnotherArray> anotherArray; + + public MyArray(Builder builder) { + this(builder, true); + } + + private MyArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myArray[] must be initialized: " + builder.__uninitialized); + + intVal = (builder.intVal == null) ? + new IntegerNode(14) : new IntegerNode(builder.intVal); + stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode()); + enumVal = (builder.enumVal == null) ? + new EnumVal(EnumVal.TYPE) : new EnumVal(builder.enumVal); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + anotherArray = AnotherArray.createVector(builder.anotherArray); + } + + /** + * @return allfeatures.myArray[].intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.myArray[].stringVal[] + */ + public List<String> stringVal() { + return stringVal.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[].stringVal[] + */ + public String stringVal(int i) { + return stringVal.get(i).value(); + } + + /** + * @return allfeatures.myArray[].enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.myArray[].refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.myArray[].anotherArray[] + */ + public List<AnotherArray> anotherArray() { + return anotherArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[].anotherArray[] + */ + public AnotherArray anotherArray(int i) { + return anotherArray.get(i); + } + + private ChangesRequiringRestart getChangesRequiringRestart(MyArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("myArray"); + return changes; + } + + private static InnerNodeVector<MyArray> createVector(List<Builder> builders) { + List<MyArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new MyArray(b)); + } + return new InnerNodeVector<MyArray>(elems); + } + + /** + * This class represents allfeatures.myArray[].enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {INNER, ENUM, TYPE} + public final static Enum INNER = Enum.INNER; + public final static Enum ENUM = Enum.ENUM; + public final static Enum TYPE = Enum.TYPE; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.myArray[].anotherArray[] + */ + public final static class AnotherArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private Integer foo = null; + + public Builder() { } + + public Builder(AnotherArray config) { + foo(config.foo()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + return this; + } + + public Builder foo(int __value) { + foo = __value; + return this; + } + + private Builder foo(String __value) { + return foo(Integer.valueOf(__value)); + } + } + + private final IntegerNode foo; + + public AnotherArray(Builder builder) { + this(builder, true); + } + + private AnotherArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myArray[].anotherArray[] must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new IntegerNode(-4) : new IntegerNode(builder.foo); + } + + /** + * @return allfeatures.myArray[].anotherArray[].foo + */ + public int foo() { + return foo.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray"); + return changes; + } + + private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) { + List<AnotherArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new AnotherArray(b)); + } + return new InnerNodeVector<AnotherArray>(elems); + } + } + } + + /** + * This class represents allfeatures.myMap{} + */ + public final static class MyMap extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "refVal" + )); + + private Integer intVal = null; + public List<String> stringVal = new ArrayList<>(); + private EnumVal.Enum enumVal = null; + private String refVal = null; + public List<AnotherArray.Builder> anotherArray = new ArrayList<>(); + + public Builder() { } + + public Builder(MyMap config) { + intVal(config.intVal()); + stringVal(config.stringVal()); + enumVal(config.enumVal()); + refVal(config.refVal()); + for (AnotherArray a : config.anotherArray()) { + anotherArray(new AnotherArray.Builder(a)); + } + } + + private Builder override(Builder __superior) { + if (__superior.intVal != null) + intVal(__superior.intVal); + if (!__superior.stringVal.isEmpty()) + stringVal.addAll(__superior.stringVal); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (!__superior.anotherArray.isEmpty()) + anotherArray.addAll(__superior.anotherArray); + return this; + } + + public Builder intVal(int __value) { + intVal = __value; + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder stringVal(String __value) { + stringVal.add(__value); + return this; + } + + public Builder stringVal(Collection<String> __values) { + stringVal.addAll(__values); + return this; + } + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + /** + * Add the given builder to this builder's list of AnotherArray builders + * @param __builder a builder + * @return this builder + */ + public Builder anotherArray(AnotherArray.Builder __builder) { + anotherArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of AnotherArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder anotherArray(List<AnotherArray.Builder> __builders) { + anotherArray = __builders; + return this; + } + } + + private final IntegerNode intVal; + private final LeafNodeVector<String, StringNode> stringVal; + private final EnumVal enumVal; + private final ReferenceNode refVal; + private final InnerNodeVector<AnotherArray> anotherArray; + + public MyMap(Builder builder) { + this(builder, true); + } + + private MyMap(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myMap{} must be initialized: " + builder.__uninitialized); + + intVal = (builder.intVal == null) ? + new IntegerNode(15) : new IntegerNode(builder.intVal); + stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode()); + enumVal = (builder.enumVal == null) ? + new EnumVal(EnumVal.ENUM) : new EnumVal(builder.enumVal); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + anotherArray = AnotherArray.createVector(builder.anotherArray); + } + + /** + * @return allfeatures.myMap{}.intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.myMap{}.stringVal[] + */ + public List<String> stringVal() { + return stringVal.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.myMap{}.stringVal[] + */ + public String stringVal(int i) { + return stringVal.get(i).value(); + } + + /** + * @return allfeatures.myMap{}.enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.myMap{}.refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.myMap{}.anotherArray[] + */ + public List<AnotherArray> anotherArray() { + return anotherArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myMap{}.anotherArray[] + */ + public AnotherArray anotherArray(int i) { + return anotherArray.get(i); + } + + private ChangesRequiringRestart getChangesRequiringRestart(MyMap newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("myMap"); + return changes; + } + + private static Map<String, MyMap> createMap(Map<String, Builder> builders) { + Map<String, MyMap> ret = new LinkedHashMap<>(); + for(String key : builders.keySet()) { + ret.put(key, new MyMap(builders.get(key))); + } + return Collections.unmodifiableMap(ret); + } + + /** + * This class represents allfeatures.myMap{}.enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {INNER, ENUM, TYPE} + public final static Enum INNER = Enum.INNER; + public final static Enum ENUM = Enum.ENUM; + public final static Enum TYPE = Enum.TYPE; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.myMap{}.anotherArray[] + */ + public final static class AnotherArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private Integer foo = null; + + public Builder() { } + + public Builder(AnotherArray config) { + foo(config.foo()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + return this; + } + + public Builder foo(int __value) { + foo = __value; + return this; + } + + private Builder foo(String __value) { + return foo(Integer.valueOf(__value)); + } + } + + private final IntegerNode foo; + + public AnotherArray(Builder builder) { + this(builder, true); + } + + private AnotherArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myMap{}.anotherArray[] must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new IntegerNode(-5) : new IntegerNode(builder.foo); + } + + /** + * @return allfeatures.myMap{}.anotherArray[].foo + */ + public int foo() { + return foo.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray"); + return changes; + } + + private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) { + List<AnotherArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new AnotherArray(b)); + } + return new InnerNodeVector<AnotherArray>(elems); + } + } + } + +} diff --git a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala b/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala deleted file mode 100644 index c1a5eb2dd6a..00000000000 --- a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import org.junit.Assert.assertThat -import org.junit.Assert.assertTrue -import org.hamcrest.CoreMatchers.is -import java.io.StringReader -import ConfiggenUtil.createClassName -import JavaClassBuilder.createUniqueSymbol -import org.junit.{Ignore, Test} - -/** - * @author gjoranv - */ -class JavaClassBuilderTest { - - @Ignore - @Test - def visual_inspection_of_generated_class() { - val testDefinition = - """version=1 - |namespace=test - |p path - |pathArr[] path - |f file - |fileArr[] file - |i int default=0 - |# A long value - |l long default=0 - |s string default="" - |b bool - |# An enum value - |e enum {A, B, C} - |intArr[] int - |boolArr[] bool - |enumArr[] enum {FOO, BAR} - |intMap{} int - |# A struct - |# with multi-line - |# comment and "quotes". - |myStruct.i int - |myStruct.s string - |# An inner array - |myArr[].i int - |myArr[].newStruct.s string - |myArr[].newStruct.b bool - |myArr[].intArr[] int - |# An inner map - |myMap{}.i int - |myMap{}.newStruct.s string - |myMap{}.newStruct.b bool - |myMap{}.intArr[] int - |intMap{} int - |""".stripMargin - - val parser = new DefParser("test", new StringReader(testDefinition)) - val root = parser.getTree - val builder = new JavaClassBuilder(root, parser.getNormalizedDefinition, null, null) - val configClass = builder.getConfigClass("TestConfig") - print(configClass) - } - - @Test - def testCreateUniqueSymbol() { - val testDefinition = - """version=1 - |namespace=test - |m int - |n int - """.stripMargin - val root = new DefParser("test", new StringReader(testDefinition)).getTree - - assertThat(createUniqueSymbol(root, "foo"), is("f")) - assertThat(createUniqueSymbol(root, "name"), is("na")) - assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")) - - // The basis string is not a legal return value, even if unique, to avoid multiple symbols - // with the same name if the same basis string is given twice. - assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my")) - } - - @Test - def testCreateClassName() { - assertThat(createClassName("simple"), is("SimpleConfig")) - assertThat(createClassName("a"), is("AConfig")) - assertThat(createClassName("a-b-c"), is("ABCConfig")) - assertThat(createClassName("a-1-2b"), is("A12bConfig")) - assertThat(createClassName("my-app"), is("MyAppConfig")) - assertThat(createClassName("MyApp"), is("MyAppConfig")) - } - - @Test(expected=classOf[CodegenRuntimeException]) - def testIllegalClassName() { - createClassName("+illegal") - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 549af4d1f64..f8e1bcec0c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; @@ -50,11 +51,15 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,6 +73,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.nio.file.Files.readAttributes; + /** * The API for managing applications. * @@ -153,7 +160,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, boolean ignoreLockFailure, boolean ignoreSessionStaleFailure, Instant now) { - return deploy(decompressApplication(in), prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); + File tempDir = Files.createTempDir(); + PrepareResult prepareResult; + try { + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); + } finally { + cleanupTempDirectory(tempDir); + } + return prepareResult; } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { @@ -170,6 +184,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye /** * Creates a new deployment from the active application, if available. + * This is used for system internal redeployments, not on application package changes. * * @param application the active application to be redeployed * @return a new deployment from the local active, or empty if a local active application @@ -182,6 +197,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye /** * Creates a new deployment from the active application, if available. + * This is used for system internal redeployments, not on application package changes. * * @param application the active application to be redeployed * @param timeout the timeout to use for each individual deployment operation @@ -196,7 +212,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LocalSession activeSession = getActiveSession(tenant, application); if (activeSession == null) return Optional.empty(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, timeoutBudget); + LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, true, timeoutBudget); tenant.getLocalSessionRepo().addSession(newSession); // Keep manually deployed tenant applications on the latest version, don't change version otherwise @@ -262,29 +278,62 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return true; } - public HttpResponse clusterControllerStatusPage(Tenant tenant, ApplicationId applicationId, String hostName, String pathSuffix) { - Application application = getApplication(tenant, applicationId); - + public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. String relativePath = "clustercontroller-status/" + pathSuffix; - return httpProxy.get(application, hostName, "container-clustercontroller", relativePath); + return httpProxy.get(getApplication(applicationId), hostName, "container-clustercontroller", relativePath); } - public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) { - return getApplication(tenant, applicationId).getApplicationGeneration(); + public Long getApplicationGeneration(ApplicationId applicationId) { + return getApplication(applicationId).getApplicationGeneration(); } public void restart(ApplicationId applicationId, HostFilter hostFilter) { hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter)); } - public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId, Duration timeout) { - Application application = getApplication(tenant, applicationId); - return fileDistributionStatus.status(application, timeout); + public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) { + return fileDistributionStatus.status(getApplication(applicationId), timeout); + } + + public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, boolean deleteFromDisk) { + if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory"); + + // Find all file references in use + Set<String> fileReferencesInUse = new HashSet<>(); + Set<ApplicationId> applicationIds = listApplications(); + applicationIds.forEach(applicationId -> fileReferencesInUse.addAll(getApplication(applicationId).getModel().fileReferences() + .stream() + .map(FileReference::value) + .collect(Collectors.toSet()))); + log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse); + + // Find those on disk that are not in use + Set<String> fileReferencesOnDisk = new HashSet<>(); + File[] filesOnDisk = fileReferencesPath.listFiles(); + if (filesOnDisk != null) + fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); + log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + + Instant instant = Instant.now().minus(Duration.ofDays(14)); + Set<String> fileReferencesToDelete = fileReferencesOnDisk + .stream() + .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) + .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) + .collect(Collectors.toSet()); + if (deleteFromDisk) { + log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete); + fileReferencesToDelete.forEach(fileReference -> { + File file = new File(fileReferencesPath, fileReference); + if ( ! IOUtils.recursiveDeleteDir(file)) + log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + }); + } + return fileReferencesToDelete; } public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { @@ -292,22 +341,37 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); } - private Application getApplication(Tenant tenant, ApplicationId applicationId) { + private Application getApplication(ApplicationId applicationId) { + Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0); return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant()); } + private Set<ApplicationId> listApplications() { + return tenantRepository.getAllTenants().stream() + .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .collect(Collectors.toSet()); + } + + private boolean isFileLastModifiedBefore(File fileReference, Instant instant) { + BasicFileAttributes fileAttributes; + try { + fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); + return fileAttributes.lastModifiedTime().toInstant().isBefore(instant); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // ---------------- Convergence ---------------------------------------------------------------- - public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceConvergenceCheck(application, hostname, uri); + public HttpResponse serviceConvergenceCheck(ApplicationId applicationId, String hostname, URI uri) { + return convergeChecker.serviceConvergenceCheck(getApplication(applicationId), hostname, uri); } - public HttpResponse serviceListToCheckForConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceListToCheckForConfigConvergence(application, uri); + public HttpResponse serviceListToCheckForConfigConvergence(ApplicationId applicationId, URI uri) { + return convergeChecker.serviceListToCheckForConfigConvergence(getApplication(applicationId), uri); } // ---------------- Session operations ---------------------------------------------------------------- @@ -338,18 +402,28 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye throw new IllegalStateException("Session not prepared: " + sessionId); } - public long createSessionFromExisting(ApplicationId applicationId, DeployLogger logger, TimeoutBudget timeoutBudget) { + public long createSessionFromExisting(ApplicationId applicationId, + DeployLogger logger, + boolean internalRedeploy, + TimeoutBudget timeoutBudget) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); SessionFactory sessionFactory = tenant.getSessionFactory(); LocalSession fromSession = getExistingSession(tenant, applicationId); - LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, timeoutBudget); + LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); localSessionRepo.addSession(session); return session.getSessionId(); } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType) { - return createSession(applicationId, timeoutBudget, decompressApplication(in, contentType)); + File tempDir = Files.createTempDir(); + long sessionId; + try { + sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir)); + } finally { + cleanupTempDirectory(tempDir); + } + return sessionId; } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { @@ -368,6 +442,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<TenantName> tenantsToBeDeleted = tenantRepository.getAllTenantNames().stream() .filter(tenantName -> activeApplications(tenantName).isEmpty()) .filter(tenantName -> !tenantName.equals(TenantName.defaultName())) // Not allowed to remove 'default' tenant + .filter(tenantName -> !tenantName.equals(TenantRepository.HOSTED_VESPA_TENANT)) // Not allowed to remove 'hosted-vespa' tenant .collect(Collectors.toSet()); tenantsToBeDeleted.forEach(tenantRepository::deleteTenant); return tenantsToBeDeleted; @@ -440,21 +515,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return currentActiveApplicationSet; } - private File decompressApplication(InputStream in, String contentType) { + private File decompressApplication(InputStream in, String contentType, File tempDir) { try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { - return decompressApplication(application); + return decompressApplication(application, tempDir); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); } } - private File decompressApplication(CompressedApplicationInputStream in) { - File tempDir = Files.createTempDir(); + private File decompressApplication(CompressedApplicationInputStream in, File tempDir) { try { return in.decompress(tempDir); } catch (IOException e) { - cleanupTempDirectory(tempDir, logger); throw new IllegalArgumentException("Unable to decompress stream", e); } } @@ -464,7 +537,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return applicationRepo.listApplications(); } - private static void cleanupTempDirectory(File tempDir, DeployLogger logger) { + private void cleanupTempDirectory(File tempDir) { logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 33718774228..72a470cf937 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -17,8 +17,7 @@ import java.util.List; * Config server db is the maintainer of the serverdb directory containing def files and the file system sessions. * See also {@link com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs} which maintains directories per tenant. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigServerDB { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerDB.class.getName()); @@ -41,17 +40,13 @@ public class ConfigServerDB { } } - public static ConfigServerDB createTestConfigServerDb(String dbDir, String definitionsDir) { - return new ConfigServerDB(new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(dbDir) - .configDefinitionsDir(definitionsDir))); - } - // The config definitions shipped with Vespa public File classes() { return new File(Defaults.getDefaults().underVespaHome(configserverConfig.configDefinitionsDir()));} public File serverdefs() { return new File(serverDB, "serverdefs"); } + public File path() { return serverDB; } + public static void createDirectory(File d) { if (d.exists()) { if (!d.isDirectory()) { @@ -81,7 +76,4 @@ public class ConfigServerDB { } } - public ConfigserverConfig getConfigserverConfig() { - return configserverConfig; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index ea14b9194e8..6c9b7216e59 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -9,8 +9,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; /** * Contains the context for serving getconfig requests so that this information does not have to be looked up multiple times. * - * @author lulf - * @since 5.8 + * @author Ulf Lilleengen */ public class GetConfigContext { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index b6e8f013fd1..6828204b17c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -27,7 +27,6 @@ public interface GlobalComponentRegistry { Curator getCurator(); ConfigCurator getConfigCurator(); Metrics getMetrics(); - ConfigServerDB getServerDB(); SessionPreparer getSessionPreparer(); ConfigserverConfig getConfigserverConfig(); TenantListener getTenantListener(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index d0b830aceaa..88f54e569df 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -31,7 +31,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry private final ConfigCurator configCurator; private final Metrics metrics; private final ModelFactoryRegistry modelFactoryRegistry; - private final ConfigServerDB serverDB; private final SessionPreparer sessionPreparer; private final RpcServer rpcServer; private final ConfigserverConfig configserverConfig; @@ -47,7 +46,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, - ConfigServerDB serverDB, SessionPreparer sessionPreparer, RpcServer rpcServer, ConfigserverConfig configserverConfig, @@ -61,7 +59,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry this.configCurator = configCurator; this.metrics = metrics; this.modelFactoryRegistry = modelFactoryRegistry; - this.serverDB = serverDB; this.sessionPreparer = sessionPreparer; this.rpcServer = rpcServer; this.configserverConfig = configserverConfig; @@ -80,8 +77,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public Metrics getMetrics() { return metrics; } @Override - public ConfigServerDB getServerDB() { return serverDB; } - @Override public SessionPreparer getSessionPreparer() { return sessionPreparer; } @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 8e865f96db3..f723ca2fe91 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -51,7 +51,7 @@ public class SuperModelController { ConfigKey<?> configKey = request.getConfigKey(); InnerCNode targetDef = getConfigDefinition(request.getConfigKey(), request.getDefContent()); ConfigPayload payload = model.getConfig(configKey); - return responseFactory.createResponse(payload, targetDef, generation); + return responseFactory.createResponse(payload, targetDef, generation, false); } private InnerCNode getConfigDefinition(ConfigKey<?> configKey, DefContent defContent) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index a73fc95eb05..64123420622 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -32,23 +32,26 @@ import java.util.Set; * a Vespa application, i.e. generation, model and zookeeper data, as well as methods for resolving config * and other queries against the model. * - * @author Harald Musum + * @author hmusum */ public class Application implements ModelResult { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Application.class.getName()); private final long appGeneration; // The generation of the set of configs belonging to an application + private final boolean internalRedeploy; private final Version vespaVersion; private final Model model; private final ServerCache cache; private final MetricUpdater metricUpdater; private final ApplicationId app; - public Application(Model model, ServerCache cache, long appGeneration, Version vespaVersion, MetricUpdater metricUpdater, ApplicationId app) { + public Application(Model model, ServerCache cache, long appGeneration, boolean internalRedeploy, + Version vespaVersion, MetricUpdater metricUpdater, ApplicationId app) { Objects.requireNonNull(model, "The model cannot be null"); this.model = model; this.cache = cache; this.appGeneration = appGeneration; + this.internalRedeploy = internalRedeploy; this.vespaVersion = vespaVersion; this.metricUpdater = metricUpdater; this.app = app; @@ -106,7 +109,7 @@ public class Application implements ModelResult { debug("Resolving config " + cacheKey); } - if (!req.noCache()) { + if ( ! req.noCache()) { ConfigResponse config = cache.get(cacheKey); if (config != null) { if (logDebug()) { @@ -131,9 +134,9 @@ public class Application implements ModelResult { throw new ConfigurationRuntimeException("Unable to resolve config " + configKey); } - ConfigResponse configResponse = responseFactory.createResponse(payload, def.getCNode(), appGeneration); + ConfigResponse configResponse = responseFactory.createResponse(payload, def.getCNode(), appGeneration, internalRedeploy); metricUpdater.incrementProcTime(System.currentTimeMillis() - start); - if (!req.noCache()) { + if ( ! req.noCache()) { cache.put(cacheKey, configResponse, configResponse.getConfigMd5()); metricUpdater.setCacheConfigElems(cache.configElems()); metricUpdater.setCacheChecksumElems(cache.checkSumElems()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java index 93cd68f6dd6..293f35558cb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -12,23 +12,24 @@ import java.io.File; /* * Holds file system directories for a tenant * - * @author tonytv + * @author Tony Vaagenes */ public class TenantFileSystemDirs { private final File serverDB; private final TenantName tenant; + public TenantFileSystemDirs(ConfigserverConfig configserverConfig, TenantName tenant) { + this(new ConfigServerDB(configserverConfig).path(), tenant); + } + + // For testing public TenantFileSystemDirs(File dir, TenantName tenant) { this.serverDB = dir; this.tenant = tenant; ConfigServerDB.createDirectory(sessionsPath()); } - public static TenantFileSystemDirs createTestDirs(TenantName tenantName) { - return new TenantFileSystemDirs(Files.createTempDir(), tenantName); - } - public File sessionsPath() { return new File(serverDB, Path.fromString("tenants").append(tenant.value()).append("sessions").getRelative()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index 22ce952481d..68c129d1e42 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -13,7 +13,7 @@ import java.util.Map; * Interface for initializing zookeeper and deploying an application package to zookeeper. * Initialize must be called before each deploy. * - * @author lulf + * @author Ulf Lilleengen */ public class ZooKeeperDeployer { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 1c2c24cc7bb..df2287c64cb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -26,10 +26,6 @@ public class FileDirectory { private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); private final File root; - public FileDirectory() { - this(FileDistribution.getDefaultFileDBPath()); - } - public FileDirectory(File rootDir) { root = rootDir; try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index bfc195cb32e..2db89c2e8ed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.jrt.ErrorCode; @@ -9,8 +10,11 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import java.io.File; import java.util.Set; import java.util.logging.Logger; @@ -20,10 +24,11 @@ import java.util.logging.Logger; public class FileDistributionImpl implements FileDistribution { private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName()); - private final Supervisor supervisor; + private final Supervisor supervisor = new Supervisor(new Transport()); + private final File fileReferencesDir; - FileDistributionImpl(Supervisor supervisor) { - this.supervisor = supervisor; + public FileDistributionImpl(ConfigserverConfig configserverConfig) { + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); } @Override @@ -31,6 +36,11 @@ public class FileDistributionImpl implements FileDistribution { startDownloadingFileReferences(hostName, port, fileReferences); } + @Override + public File getFileReferencesDir() { + return fileReferencesDir; + } + // Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed, // as downloading will then start synchronously when a service requests a file reference instead private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index 9cbc842d8c2..b3f3214793c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.jrt.Supervisor; import java.io.File; @@ -17,13 +16,11 @@ public class FileDistributionProvider { private final FileRegistry fileRegistry; private final FileDistribution fileDistribution; - public FileDistributionProvider(Supervisor supervisor, File applicationDir) { - ensureDirExists(FileDistribution.getDefaultFileDBPath()); - this.fileDistribution = new FileDistributionImpl(supervisor); - this.fileRegistry = new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory())); + public FileDistributionProvider(File applicationDir, FileDistribution fileDistribution) { + this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory(fileDistribution.getFileReferencesDir()))), fileDistribution); + ensureDirExists(fileDistribution.getFileReferencesDir()); } - // For testing only FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) { this.fileRegistry = fileRegistry; this.fileDistribution = fileDistribution; @@ -37,7 +34,7 @@ public class FileDistributionProvider { return fileDistribution; } - private void ensureDirExists(File dir) { + private static void ensureDirExists(File dir) { if (!dir.exists()) { boolean success = dir.mkdirs(); if (!success) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 001ef751e69..42bf269e9d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -17,6 +16,7 @@ import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; @@ -72,7 +72,7 @@ public class FileServer { @Inject public FileServer(ConfigserverConfig configserverConfig) { - this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath()); + this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()))); } // For testing only diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java index 728f327c829..40d75d9dbac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java @@ -4,14 +4,23 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; +import java.io.File; import java.util.Set; /** * @author Ulf Lilleengen */ -public class MockFileDBHandler implements FileDistribution { +public class MockFileDistribution implements FileDistribution { + private final File fileReferencesDir; + + MockFileDistribution(File fileReferencesDir) { + this.fileReferencesDir = fileReferencesDir; + } @Override public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {} + @Override + public File getFileReferencesDir() { return fileReferencesDir; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java index 531ba388d00..db70a51b2b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -4,15 +4,16 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.application.provider.MockFileRegistry; +import java.io.File; + /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class MockFileDistributionProvider extends FileDistributionProvider { public int timesCalled = 0; - public MockFileDistributionProvider() { - super(new MockFileRegistry(), new MockFileDBHandler()); + public MockFileDistributionProvider(File fileReferencesDir) { + super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir)); } public FileDistribution getFileDistribution() { @@ -20,7 +21,4 @@ public class MockFileDistributionProvider extends FileDistributionProvider { return super.getFileDistribution(); } - public MockFileDBHandler getMockFileDBHandler() { - return (MockFileDBHandler) getFileDistribution(); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 42fdb16c7ca..6bca8b1c562 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -59,13 +59,13 @@ public class ApplicationHandler extends HttpHandler { Tenant tenant = verifyTenantAndApplication(applicationId); if (isServiceConvergeRequest(request)) { - return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostNameFromRequest(request), request.getUri()); + return applicationRepository.serviceConvergenceCheck(applicationId, getHostNameFromRequest(request), request.getUri()); } if (isClusterControllerStatusRequest(request)) { String hostName = getHostNameFromRequest(request); String pathSuffix = getPathSuffix(request); - return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix); + return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix); } if (isContentRequest(request)) { @@ -86,15 +86,15 @@ public class ApplicationHandler extends HttpHandler { } if (isServiceConvergeListRequest(request)) { - return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri()); + return applicationRepository.serviceListToCheckForConfigConvergence(applicationId, request.getUri()); } if (isFiledistributionStatusRequest(request)) { Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); - return applicationRepository.filedistributionStatus(tenant, applicationId, timeout); + return applicationRepository.filedistributionStatus(applicationId, timeout); } - return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId)); + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 13933544ad1..d8385456866 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.log.LogLevel; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 74c85829ef2..26ef79ebe02 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -58,7 +58,7 @@ public class SessionCreateHandler extends SessionHandler { long sessionId; if (request.hasProperty("from")) { ApplicationId applicationId = getFromApplicationId(request); - sessionId = applicationRepository.createSessionFromExisting(applicationId, logger, timeoutBudget); + sessionId = applicationRepository.createSessionFromExisting(applicationId, logger, false, timeoutBudget); } else { validateDataAndHeader(request); String name = getNameProperty(request, logger); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index c8b3bc824a8..a08b077699c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -3,7 +3,10 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.curator.Curator; import java.time.Duration; @@ -11,19 +14,43 @@ import java.time.Duration; public class ConfigServerMaintenance extends AbstractComponent { private final TenantsMaintainer tenantsMaintainer; + private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; + private final FileDistributionMaintainer fileDistributionMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, - Curator curator) { - tenantsMaintainer = new TenantsMaintainer(applicationRepository, - curator, - Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes())); + Curator curator, + FileDistributionFactory fileDistributionFactory) { + DefaultTimes defaults = new DefaultTimes(configserverConfig); + tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); + zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval); + fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); } @Override public void deconstruct() { tenantsMaintainer.deconstruct(); + zooKeeperDataMaintainer.deconstruct(); + fileDistributionMaintainer.deconstruct(); + } + + /* + * Default values from config. If one of the values needs to be changed, add the value to + * configserver-config.xml in the config server application directory and restart the config server + */ + private static class DefaultTimes { + + private final Duration defaultInterval; + private final Duration tenantsMaintainerInterval; + + DefaultTimes(ConfigserverConfig configserverConfig) { + boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); + + this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); + // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD + this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); + } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java new file mode 100644 index 00000000000..58141a3a045 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.time.Duration; + +public class FileDistributionMaintainer extends Maintainer { + + private final ApplicationRepository applicationRepository; + private final File fileReferencesDir; + + public FileDistributionMaintainer(ApplicationRepository applicationRepository, + Curator curator, + Duration interval, + ConfigserverConfig configserverConfig) { + super(applicationRepository, curator, interval); + this.applicationRepository = applicationRepository; + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));; + } + + + @Override + protected void maintain() { + // TODO: Does not delete, for now just outputs what should be deleted + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java index ce0811184a3..ae000385dfd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java @@ -20,7 +20,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); private static final Path root = Path.fromString("/configserver/v1/"); - private static final com.yahoo.path.Path lockRoot = root.append("locks"); + private static final Path lockRoot = root.append("locks"); private final Duration maintenanceInterval; private final ScheduledExecutorService service; @@ -36,13 +36,10 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } @Override - @SuppressWarnings("try") + @SuppressWarnings({"try", "unused"}) public void run() { - try { - Path path = lockRoot.append(name()); - try (Lock lock = new Lock(path.toString(), curator)) { - maintain(); - } + try (Lock lock = lock(lockRoot.append(name()))) { + maintain(); } catch (UncheckedTimeoutException e) { // another config server instance is running this job at the moment; ok } catch (Throwable t) { @@ -50,6 +47,12 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } } + private Lock lock(Path path) { + Lock lock = new Lock(path.getAbsolute(), curator); + lock.acquire(Duration.ofSeconds(1)); + return lock; + } + @Override public void deconstruct() { this.service.shutdown(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index e06bf530486..36306dbdde8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -8,7 +8,7 @@ import java.time.Duration; public class TenantsMaintainer extends Maintainer { - public TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { super(applicationRepository, curator, interval); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java new file mode 100644 index 00000000000..fd57732eb30 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +/** + * Removes unused zookeeper data (for now only data used by old file distribution code is removed) + */ +public class ZooKeeperDataMaintainer extends Maintainer { + + ZooKeeperDataMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, interval); + } + + @Override + protected void maintain() { + curator.delete(Path.fromString("/vespa/filedistribution")); + curator.delete(Path.fromString("/vespa/config")); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 9d583d27341..d83548dcc3d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -88,7 +88,9 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { new com.yahoo.component.Version(modelFactory.getVersion().toString()), wantedNodeVespaVersion); MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId)); - return new Application(modelFactory.createModel(modelContext), cache, appGeneration, modelFactory.getVersion(), + return new Application(modelFactory.createModel(modelContext), cache, appGeneration, + applicationPackage.getMetaData().isInternalRedeploy(), + modelFactory.getVersion(), applicationMetricUpdater, applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 3010f1383da..9d6247c5f80 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.HostProvisioner; @@ -26,6 +27,7 @@ import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.net.URI; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -108,7 +110,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { catch (RuntimeException e) { boolean isOldestMajor = i == majorVersions.size() - 1; if (isOldestMajor) { - if (e instanceof NullPointerException || e instanceof NoSuchElementException) { + if (e instanceof NullPointerException || e instanceof NoSuchElementException | e instanceof UncheckedTimeoutException) { log.log(LogLevel.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { @@ -145,7 +147,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { List<MODELRESULT> allApplicationVersions = new ArrayList<>(); allApplicationVersions.add(latestModelVersion); - if (zone().environment() == Environment.dev) + if (Arrays.asList(Environment.dev, Environment.test).contains(zone().environment())) versions = keepThoseUsedOn(allocatedHosts.get(), versions); // TODO: We use the allocated hosts from the newest version when building older model versions. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java index 07e07fa2595..247ae388639 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java @@ -9,17 +9,19 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; * Represents a component that creates config responses from a payload. Different implementations * can do transformations of the payload such as compression. * - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ public interface ConfigResponseFactory { /** * Create a {@link ConfigResponse} for a given payload and generation. + * * @param payload The {@link com.yahoo.vespa.config.ConfigPayload} to put in the response. * @param defFile The {@link com.yahoo.config.codegen.InnerCNode} def file for this config. * @param generation The payload generation. @return A {@link ConfigResponse} that can be sent to the client. + * @param internalRedeployment whether this config generation was produced by an internal redeployment, + * not a change to the application package */ - ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation); + ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation, boolean internalRedeployment); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index b37d013cc6b..970fe9f169b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -22,7 +22,6 @@ import java.util.logging.Logger; /** * @author hmusum -* @since 5.1 */ class GetConfigProcessor implements Runnable { @@ -30,8 +29,9 @@ class GetConfigProcessor implements Runnable { private static final String localHostName = HostName.getLocalhost(); private final JRTServerConfigRequest request; + /* True only when this request has expired its server timeout and we need to respond to the client */ - private boolean forceResponse = false; + private final boolean forceResponse; private final RpcServer rpcServer; private String logPre = ""; @@ -64,7 +64,7 @@ class GetConfigProcessor implements Runnable { // TODO: Increment statistics (Metrics) failed counters when requests fail public void run() { //Request has already been detached - if (!request.validateParameters()) { + if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); @@ -121,7 +121,7 @@ class GetConfigProcessor implements Runnable { // config == null is not an error, but indicates that the config will be returned later. if ((config != null) && (!config.hasEqualConfig(request) || config.hasNewerGeneration(request) || forceResponse)) { // debugLog(trace, "config response before encoding:" + config.toString()); - request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.getConfigMd5()); + request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.isInternalRedeploy(), config.getConfigMd5()); if (logDebug(trace)) { debugLog(trace, "return response: " + request.getShortDescription()); } @@ -146,8 +146,8 @@ class GetConfigProcessor implements Runnable { private void returnEmpty(JRTServerConfigRequest request) { ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); - ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, configMd5); - request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.getConfigMd5()); + ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5); + request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5()); respond(request); } @@ -161,4 +161,5 @@ class GetConfigProcessor implements Runnable { trace.trace(RpcServer.TRACELEVEL_DEBUG, logPre + message); } } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java index 609f1d5f79f..ff15eb7bfa7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java @@ -14,19 +14,22 @@ import com.yahoo.vespa.config.util.ConfigUtils; /** * Compressor that compresses config payloads to lz4. * - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ public class LZ4ConfigResponseFactory implements ConfigResponseFactory { private static LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); @Override - public ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation) { + public ConfigResponse createResponse(ConfigPayload payload, + InnerCNode defFile, + long generation, + boolean internalRedeployment) { Utf8Array rawPayload = payload.toUtf8Array(true); String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes())); - return new SlimeConfigResponse(compressed, defFile, generation, configMd5, info); + return new SlimeConfigResponse(compressed, defFile, generation, internalRedeployment, configMd5, info); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java index ac3cfa2fda1..995e981e0f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java @@ -13,17 +13,19 @@ import com.yahoo.vespa.config.util.ConfigUtils; /** * Simply returns an uncompressed payload. * - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ public class UncompressedConfigResponseFactory implements ConfigResponseFactory { @Override - public ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation) { + public ConfigResponse createResponse(ConfigPayload payload, + InnerCNode defFile, + long generation, + boolean internalRedeployment) { Utf8Array rawPayload = payload.toUtf8Array(true); String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); - return new SlimeConfigResponse(rawPayload, defFile, generation, configMd5, info); + return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeployment, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java index 32602ab70b8..8394494adca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import java.io.File; @@ -15,16 +16,15 @@ import java.io.File; @SuppressWarnings("WeakerAccess") public class FileDistributionFactory { - private final Supervisor supervisor = new Supervisor(new Transport()); + private final ConfigserverConfig configserverConfig; - public FileDistributionProvider createProvider(File applicationPackage) { - return new FileDistributionProvider(supervisor, applicationPackage); + @Inject + public FileDistributionFactory(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; } - @Override - @SuppressWarnings("deprecation") // finalize() is deprecated from Java 9 - protected void finalize() throws Throwable { - super.finalize(); - supervisor.transport().shutdown().join(); + public FileDistributionProvider createProvider(File applicationPackage) { + return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig)); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 653df81f296..3978a1f25f8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -9,8 +9,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; * class represents the common stuff between sessions working on the local file * system ({@link LocalSession}s) and sessions working on zookeeper {@link RemoteSession}s. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public abstract class Session { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java index bd9da36a2ba..a3dea83d50c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java @@ -10,7 +10,7 @@ import java.io.File; /** * A session factory responsible for creating deploy sessions. * - * @author lulf + * @author Ulf Lilleengen */ public interface SessionFactory { @@ -31,9 +31,11 @@ public interface SessionFactory { * * @param existingSession The session to use as base * @param logger a deploy logger where the deploy log will be written. + * @param internalRedeploy if this session is for a system internal redeploy not an application package change * @param timeoutBudget Timeout for creating session and waiting for other servers. * @return a new session */ - LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, TimeoutBudget timeoutBudget); + LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, + boolean internalRedeploy, TimeoutBudget timeoutBudget); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 7285ff905ff..10590a26690 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -74,9 +74,12 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { this.clock = globalComponentRegistry.getClock(); } + /** Create a session for a true application package change */ @Override - public LocalSession createSession(File applicationFile, ApplicationId applicationId, TimeoutBudget timeoutBudget) { - return create(applicationFile, applicationId, nonExistingActiveSession, timeoutBudget); + public LocalSession createSession(File applicationFile, + ApplicationId applicationId, + TimeoutBudget timeoutBudget) { + return create(applicationFile, applicationId, nonExistingActiveSession, false, timeoutBudget); } private void ensureZKPathDoesNotExist(Path sessionPath) { @@ -89,13 +92,14 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { File configApplicationDir, String applicationName, long sessionId, - long currentlyActiveSession) { + long currentlyActiveSession, + boolean internalRedeploy) { long deployTimestamp = System.currentTimeMillis(); String user = System.getenv("USER"); if (user == null) { user = "unknown"; } - DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationName, deployTimestamp, sessionId, currentlyActiveSession); + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationName, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSession); return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); } @@ -119,19 +123,21 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { @Override public LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { File existingApp = getSessionAppDir(existingSession.getSessionId()); ApplicationId existingApplicationId = existingSession.getApplicationId(); long liveApp = getLiveApp(existingApplicationId); logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", live app for it is " + liveApp); - LocalSession session = create(existingApp, existingApplicationId, liveApp, timeoutBudget); + LocalSession session = create(existingApp, existingApplicationId, liveApp, internalRedeploy, timeoutBudget); session.setApplicationId(existingApplicationId); session.setVespaVersion(existingSession.getVespaVersion()); return session; } - private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSession, TimeoutBudget timeoutBudget) { + private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSession, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { long sessionId = sessionCounter.nextSessionId(); Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Next session id is " + sessionId + " , sessionIdPath=" + sessionIdPath.getAbsolute()); @@ -145,8 +151,12 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { nodeFlavors); File userApplicationDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); IOUtils.copyDirectory(applicationFile, userApplicationDir); - ApplicationPackage applicationPackage = createApplication(applicationFile, userApplicationDir, - applicationId.application().value(), sessionId, currentlyActiveSession); + ApplicationPackage applicationPackage = createApplication(applicationFile, + userApplicationDir, + applicationId.application().value(), + sessionId, + currentlyActiveSession, + internalRedeploy); applicationPackage.writeMetaData(); return createSessionFromApplication(applicationPackage, sessionId, sessionZooKeeperClient, timeoutBudget, clock); } catch (Exception e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index ed1eba58dfb..117930c6d7d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -23,18 +23,19 @@ import java.util.logging.Logger; public class SessionStateWatcher implements NodeCacheListener { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); + // One thread pool for all instances of this class + private static final Executor executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(SessionStateWatcher.class.getName())); + private final Curator.FileCache fileCache; private final ReloadHandler reloadHandler; private final RemoteSession session; private final MetricUpdater metrics; - private final Executor executor; + public SessionStateWatcher(Curator.FileCache fileCache, ReloadHandler reloadHandler, RemoteSession session, MetricUpdater metrics) { - executor = Executors.newSingleThreadExecutor( - ThreadFactoryFactory.getThreadFactory(SessionStateWatcher.class.getName() + "-" + session)); this.fileCache = fileCache; this.reloadHandler = reloadHandler; this.session = session; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 69721ed01d4..ad967f49964 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -14,9 +14,7 @@ import com.yahoo.vespa.config.server.application.ZKTenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.session.*; -import com.yahoo.vespa.defaults.Defaults; -import java.io.File; import java.time.Clock; import java.util.Collections; @@ -162,12 +160,7 @@ public class TenantBuilder { private void createServerDbDirs() { if (tenantFileSystemDirs == null) { - tenantFileSystemDirs = new TenantFileSystemDirs(new File( - Defaults.getDefaults().underVespaHome(componentRegistry - .getServerDB() - .getConfigserverConfig() - .configServerDBDir())), - tenant); + tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigserverConfig(), tenant); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java index 8c774fd35ce..b19f8bf8d0b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java @@ -7,8 +7,7 @@ import com.yahoo.vespa.config.server.RequestHandler; /** * Represents something that can provide request and reload handlers of a tenant. * - * @author lulf - * @since 5.3 + * @author Ulf Lilleengen */ public interface TenantHandlerProvider { diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b984ce60702..2eeefda63e7 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -10,7 +10,7 @@ <initialStatus>initializing</initialStatus> </config> - <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" rotationScheme="date" compressOnRotation="true" symlinkName="access.log" /> <preprocess:include file='access-logging.xml' required='false' /> <component id="com.yahoo.vespa.config.server.ConfigServerBootstrap" bundle="configserver" /> @@ -53,8 +53,8 @@ <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> <preprocess:include file='hosted-vespa/scoreboard.xml' required='false' /> <preprocess:include file='controller/container.xml' required='false' /> - <component id="com.yahoo.vespa.service.monitor.internal.SlobrokMonitorManagerImpl" bundle="service-monitor" /> - <component id="com.yahoo.vespa.service.monitor.internal.HealthMonitorManager" bundle="service-monitor" /> + <component id="com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl" bundle="service-monitor" /> + <component id="com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.monitor.internal.ServiceMonitorImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 058e39eea9b..f7e76c2b3bb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -10,10 +10,9 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -21,14 +20,17 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collections; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,18 +50,22 @@ public class ApplicationRepositoryTest { private Tenant tenant; private ApplicationRepository applicationRepository; + private TenantRepository tenantRepository; private TimeoutBudget timeoutBudget; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setup() { Curator curator = new MockCurator(); - TenantRepository tenants = new TenantRepository(new TestComponentRegistry.Builder() + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() .curator(curator) .build()); - tenants.addTenant(tenantName); - tenant = tenants.getTenant(tenantName); + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenants, provisioner, clock); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -79,15 +85,15 @@ public class ApplicationRepositoryTest { } @Test - public void createAndPrepareAndActivate() throws IOException { - PrepareResult result = deployApp(); + public void createAndPrepareAndActivate() { + PrepareResult result = deployApp(testApp); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); } @Test - public void deleteUnusedTenants() throws IOException { - deployApp(); + public void deleteUnusedTenants() { + deployApp(testApp); assertTrue(applicationRepository.removeUnusedTenants().isEmpty()); applicationRepository.remove(applicationId()); assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next()); @@ -110,17 +116,58 @@ public class ApplicationRepositoryTest { assertEquals(Vtag.currentVersion, ApplicationRepository.decideVersion(regularApp, Environment.perf, targetVersion)); } + @Test + public void deleteUnusedFileReferences() throws IOException { + File fileReferencesDir = temporaryFolder.newFolder(); + + // Add file reference that is not in use and should be deleted (older than 14 days) + File filereferenceDir = createFilereferenceOnDisk(new File(fileReferencesDir, "foo"), Instant.now().minus(Duration.ofDays(15))); + // Add file reference that is not in use, but should not be deleted (not older than 14 days) + File filereferenceDir2 = createFilereferenceOnDisk(new File(fileReferencesDir, "baz"), Instant.now()); + + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); + Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); + + // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now + PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); + deployApp(new File("src/test/apps/app"), prepareParams); + + boolean deleteFiles = false; + Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(Collections.singleton("foo"), toBeDeleted); + assertTrue(filereferenceDir.exists()); + assertTrue(filereferenceDir2.exists()); + + deleteFiles = true; + toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(Collections.singleton("foo"), toBeDeleted); + assertFalse(filereferenceDir.exists()); + assertTrue(filereferenceDir2.exists()); + } + + private File createFilereferenceOnDisk(File filereferenceDir, Instant lastModifiedTime) { + assertTrue(filereferenceDir.mkdir()); + File bar = new File(filereferenceDir, "file"); + IOUtils.writeFile(bar, Utf8.toBytes("test")); + assertTrue(filereferenceDir.setLastModified(lastModifiedTime.toEpochMilli())); + return filereferenceDir; + } + private PrepareResult prepareAndActivateApp(File application) throws IOException { FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir()); return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now()); } - private PrepareResult deployApp() throws IOException { - File file = CompressedApplicationInputStreamTest.createTarFile(); - return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream( - new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP), - prepareParams(), false, false, Instant.now()); + private PrepareResult deployApp(File applicationPackage) { + return deployApp(applicationPackage, prepareParams()); + } + + private PrepareResult deployApp(File applicationPackage, PrepareParams prepareParams) { + return applicationRepository.deploy(applicationPackage, prepareParams); } private PrepareParams prepareParams() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 77550b3cd35..67cc87ae223 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; @@ -9,12 +8,12 @@ import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; @@ -35,7 +34,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootStrap() throws Exception { - ConfigserverConfig configserverConfig = createConfigserverConfig(); + ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); tester.deployApp("myApp", "4.5.6", Instant.now()); @@ -55,7 +54,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootStrapWhenRedeploymentFails() throws Exception { - ConfigserverConfig configserverConfig = createConfigserverConfig(); + ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); tester.deployApp("myApp", "4.5.6", Instant.now()); @@ -90,8 +89,8 @@ public class ConfigServerBootstrapTest { throw new RuntimeException(messageIfWaitingFails); } - private MockRpc createRpcServer(ConfigserverConfig configserverConfig) { - return new MockRpc(configserverConfig.rpcport()); + private MockRpc createRpcServer(ConfigserverConfig configserverConfig) throws IOException { + return new MockRpc(configserverConfig.rpcport(), temporaryFolder.newFolder()); } private StateMonitor createStateMonitor() { @@ -99,10 +98,10 @@ public class ConfigServerBootstrapTest { new SystemTimer()); } - private static ConfigserverConfig createConfigserverConfig() { + private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder) throws IOException { return new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) .hostedVespa(true) .multitenant(true)); } @@ -111,8 +110,8 @@ public class ConfigServerBootstrapTest { volatile boolean isRunning = false; - MockRpc(int port) { - super(port); + MockRpc(int port, File tempDir) { + super(port, tempDir); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java index c34f6512f29..0c8a9e5c3d8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.io.IOUtils; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -13,25 +15,28 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigServerDBTest { private ConfigServerDB serverDB; - private File dbDir; - private File definitionsDir; + private ConfigserverConfig configserverConfig; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setup() { - dbDir = Files.createTempDir(); - definitionsDir = Files.createTempDir(); - serverDB = ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath()); + public void setup() throws IOException { + configserverConfig = new ConfigserverConfig( + new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); + serverDB = new ConfigServerDB(configserverConfig); } private void createInitializer() throws IOException { File existingDef = new File(serverDB.classes(), "test.def"); IOUtils.writeFile(existingDef, "hello", false); - ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath()); + new ConfigServerDB(configserverConfig); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index dec9dd991de..5ca3deab1fe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.filedistribution.FileServer; @@ -22,8 +20,11 @@ import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.util.Collections; import static org.hamcrest.Matchers.is; @@ -31,14 +32,12 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InjectedGlobalComponentRegistryTest { private Curator curator; private Metrics metrics; - private ConfigServerDB serverDB; private SessionPreparer sessionPreparer; private ConfigserverConfig configserverConfig; private RpcServer rpcServer; @@ -50,33 +49,36 @@ public class InjectedGlobalComponentRegistryTest { private ModelFactoryRegistry modelFactoryRegistry; private Zone zone; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRegistry() { + public void setupRegistry() throws IOException { curator = new MockCurator(); ConfigCurator configCurator = ConfigCurator.create(curator); metrics = Metrics.createTestMetrics(); modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath())); - serverDB = new ConfigServerDB(configserverConfig); + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); sessionPreparer = new SessionTest.MockSessionPreparer(); - rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences"))); generationCounter = new SuperModelGenerationCounter(curator); defRepo = new StaticConfigDefinitionRepo(); permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); hostRegistries = new HostRegistries(); HostProvisionerProvider hostProvisionerProvider = HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner()); zone = Zone.defaultZone(); - globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, serverDB, sessionPreparer, rpcServer, configserverConfig, generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); + globalComponentRegistry = + new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, rpcServer, configserverConfig, + generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); } @Test public void testThatAllComponentsAreSetup() { assertThat(globalComponentRegistry.getModelFactoryRegistry(), is(modelFactoryRegistry)); - assertThat(globalComponentRegistry.getServerDB(), is(serverDB)); assertThat(globalComponentRegistry.getSessionPreparer(), is(sessionPreparer)); assertThat(globalComponentRegistry.getMetrics(), is(metrics)); assertThat(globalComponentRegistry.getCurator(), is(curator)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java index 2ccce9727a3..8b515ff837b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; @@ -10,7 +11,7 @@ import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import java.util.Collection; -import java.util.Optional; +import java.util.HashSet; import java.util.Set; /** @@ -39,9 +40,10 @@ public class ModelStub implements Model { } @Override - public void distributeFiles(FileDistribution fileDistribution) { + public void distributeFiles(FileDistribution fileDistribution) { } - } + @Override + public Set<FileReference> fileReferences() { return new HashSet<>(); } @Override public AllocatedHosts allocatedHosts() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java index c1db543d386..b93b3fa28e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java @@ -50,9 +50,9 @@ public class ServerCacheTest { cache.addDef(fooBimDefKey, new ConfigDefinition("mynode", new String[0])); - cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, configMd5), configMd5); - cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, configMd5), configMd5); - cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload_2.getCNode(), 2, configMd5_2), configMd5_2); + cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, false, configMd5), configMd5); + cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, false, configMd5), configMd5); + cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload_2.getCNode(), 2, false, configMd5_2), configMd5_2); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index bc07ac7d79c..2b72db63d55 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -122,12 +122,11 @@ public class SuperModelRequestHandlerTest { } private static class TestApplication extends Application { - private long version = 0; public TestApplication(VespaModel vespaModel, ServerCache cache, long appGeneration, ApplicationId app, long version) { - super(vespaModel, cache, appGeneration, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app); - this.version = version; + super(vespaModel, cache, appGeneration, false, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app); } + } public static NodeFlavors emptyNodeFlavors() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index d026989a43e..e4e45d3a014 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.model.VespaModelFactory; +import java.io.File; import java.time.Clock; import java.util.Collections; import java.util.Optional; @@ -37,7 +38,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final Curator curator; private final ConfigCurator configCurator; private final Metrics metrics; - private final ConfigServerDB serverDB; private final SessionPreparer sessionPreparer; private final ConfigserverConfig configserverConfig; private final SuperModelGenerationCounter superModelGenerationCounter; @@ -57,7 +57,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { PermanentApplicationPackage permanentApplicationPackage, FileDistributionFactory fileDistributionFactory, SuperModelGenerationCounter superModelGenerationCounter, - ConfigServerDB configServerDB, HostRegistries hostRegistries, ConfigserverConfig configserverConfig, SessionPreparer sessionPreparer, @@ -71,7 +70,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.configCurator = configCurator; this.metrics = metrics; this.configserverConfig = configserverConfig; - this.serverDB = configServerDB; this.reloadListener = reloadListener; this.tenantListener = tenantListener; this.superModelGenerationCounter = superModelGenerationCounter; @@ -155,7 +153,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory - .orElse(new MockFileDistributionFactory()); + .orElse(new MockFileDistributionFactory(new File(configserverConfig.fileReferencesDir()))); HostProvisionerProvider hostProvisionerProvider = hostProvisioner.isPresent() ? HostProvisionerProvider.withProvisioner(hostProvisioner.get()) : HostProvisionerProvider.empty(); @@ -168,7 +166,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { permApp, fileDistributionFactory, new SuperModelGenerationCounter(curator), - new ConfigServerDB(configserverConfig), hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, zone, clock); @@ -182,8 +179,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { @Override public Metrics getMetrics() { return metrics; } @Override - public ConfigServerDB getServerDB() { return serverDB; } - @Override public SessionPreparer getSessionPreparer() { return sessionPreparer; } @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java index 1d692e8f78f..3c3edae7914 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertThat; import static com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker.ServiceResponse; /** - * @author lulf + * @author Ulf Lilleengen */ public class ApplicationConvergenceCheckerTest { @@ -43,7 +43,12 @@ public class ApplicationConvergenceCheckerTest { @Before public void setup() { Model mockModel = MockModel.createContainer("localhost", 1337); - application = new Application(mockModel, new ServerCache(), 3, Version.fromIntValues(0, 0, 0), MetricUpdater.createTestUpdater(), appId); + application = new Application(mockModel, + new ServerCache(), + 3, + false, + Version.fromIntValues(0, 0, 0), + MetricUpdater.createTestUpdater(), appId); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java index a83f2676a4d..233c0e99ed8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java @@ -31,9 +31,9 @@ public class ApplicationMapperTest { vespaVersions.add(Version.fromString("1.2.3")); vespaVersions.add(Version.fromString("1.2.4")); vespaVersions.add(Version.fromString("1.2.5")); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java index 98bedb76599..94bb81021dc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java @@ -29,9 +29,9 @@ public class ApplicationSetTest { vespaVersions.add(Version.fromString("1.2.3")); vespaVersions.add(Version.fromString("1.2.4")); vespaVersions.add(Version.fromString("1.2.5")); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); - applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, false, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java index 02cf6303ba8..90a27c39736 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -42,8 +42,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.1.14 + * @author Ulf Lilleengen */ public class ApplicationTest { @@ -53,7 +52,7 @@ public class ApplicationTest { ApplicationName.from("foobar"), InstanceName.defaultName()); ServerCache cache = new ServerCache(); Version vespaVersion = Version.fromIntValues(1, 2, 3); - Application app = new Application(new ModelStub(), cache, 1337, vespaVersion, MetricUpdater.createTestUpdater(), appId); + Application app = new Application(new ModelStub(), cache, 1337L, false, vespaVersion, MetricUpdater.createTestUpdater(), appId); assertThat(app.getApplicationGeneration(), is(1337l)); assertNotNull(app.getModel()); assertThat(app.getCache(), is(cache)); @@ -71,24 +70,24 @@ public class ApplicationTest { File testApp = new File("src/test/apps/app"); ServerCache cache = createCacheAndAddContent(); VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(testApp)); - final ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("foo").build(); - handler = new Application(model, cache, 1, Version.fromIntValues(1, 2, 3), + ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("foo").build(); + handler = new Application(model, cache, 1L, false, Version.fromIntValues(1, 2, 3), new MetricUpdater(Metrics.createTestMetrics(), Metrics.createDimensions(applicationId)), applicationId); } private static ServerCache createCacheAndAddContent() { ServerCache cache = new ServerCache(); - final ConfigDefinitionKey key = new ConfigDefinitionKey(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE); + ConfigDefinitionKey key = new ConfigDefinitionKey(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE); com.yahoo.vespa.config.buildergen.ConfigDefinition def = getDef(key, SimpletypesConfig.CONFIG_DEF_SCHEMA); // TODO Why do we have to use empty def md5 here? cache.addDef(key, def); - final ConfigDefinitionKey key2 = new ConfigDefinitionKey(SlobroksConfig.CONFIG_DEF_NAME, SlobroksConfig.CONFIG_DEF_NAMESPACE); + ConfigDefinitionKey key2 = new ConfigDefinitionKey(SlobroksConfig.CONFIG_DEF_NAME, SlobroksConfig.CONFIG_DEF_NAMESPACE); com.yahoo.vespa.config.buildergen.ConfigDefinition def2 = getDef(key2, SlobroksConfig.CONFIG_DEF_SCHEMA); cache.addDef(key2, def2); - final ConfigDefinitionKey key3 = new ConfigDefinitionKey(LogdConfig.CONFIG_DEF_NAME, LogdConfig.CONFIG_DEF_NAMESPACE); + ConfigDefinitionKey key3 = new ConfigDefinitionKey(LogdConfig.CONFIG_DEF_NAME, LogdConfig.CONFIG_DEF_NAMESPACE); com.yahoo.vespa.config.buildergen.ConfigDefinition def3 = getDef(key3, LogdConfig.CONFIG_DEF_SCHEMA); cache.addDef(key3, def3); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java index 76204d5c5f2..c2ddec7e795 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java @@ -161,7 +161,13 @@ public class FileDistributionStatusTest { private Application createApplication(List<String> hostnames) { Model mockModel = MockModel.createConfigProxies(hostnames, 1337); - return new Application(mockModel, new ServerCache(), 3, Version.fromIntValues(0, 0, 0), MetricUpdater.createTestUpdater(), appId); + return new Application(mockModel, + new ServerCache(), + 3, + false, + Version.fromIntValues(0, 0, 0), + MetricUpdater.createTestUpdater(), + appId); } HttpResponse getStatus(FileDistributionStatus fileDistributionStatus, Application application) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 91b21ad83d6..65727a8b989 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; @@ -20,7 +21,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; // Model with two services, one that does not have a state port @@ -109,6 +109,9 @@ class MockModel implements Model { } @Override + public Set<FileReference> fileReferences() { return new HashSet<>(); } + + @Override public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 9b40784018a..b151ac57352 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import static com.yahoo.vespa.config.server.deploy.DeployTester.CountingModelFactory; +import com.yahoo.vespa.config.server.session.LocalSession; import org.junit.Ignore; import org.junit.Test; @@ -50,12 +51,18 @@ public class HostedDeployTest { @Test public void testRedeploy() { DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - tester.deployApp("myApp", Instant.now()); + ApplicationId appId = tester.deployApp("myApp", Instant.now()); + LocalSession s1 = tester.applicationRepository().getActiveSession(appId); + System.out.println("First session: " + s1.getSessionId()); + assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); deployment.get().prepare(); deployment.get().activate(); + LocalSession s2 = tester.applicationRepository().getActiveSession(appId); + System.out.println("Second session: " + s2.getSessionId()); + assertTrue(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index bf7f7038c1a..945c7d60750 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -43,7 +43,14 @@ public class ZooKeeperClientTest extends TestWithCurator { public void setupZK() throws IOException { this.zk = ConfigCurator.create(curator); ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), true, Path.fromString(appPath)); - ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("foo", "/bar/baz", "appName", 1345l, 3l, 2l)); + ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), + new DeployData("foo", + "/bar/baz", + "appName", + 1345L, + true, + 3L, + 2L)); Map<Version, FileRegistry> fileRegistries = createFileRegistries(); app.writeMetaData(); zkc.setupZooKeeper(); @@ -97,7 +104,7 @@ public class ZooKeeperClientTest extends TestWithCurator { // TODO: Evaluate if we want this or not @Test @Ignore - public void testFeedComponentsFileReferencesToZooKeeper() throws IOException { + public void testFeedComponentsFileReferencesToZooKeeper() { final String appDir = "src/test/apps/app_sdbundles"; ConfigCurator zk = ConfigCurator.create(new MockCurator()); BaseDeployLogger logger = new BaseDeployLogger(); @@ -120,6 +127,7 @@ public class ZooKeeperClientTest extends TestWithCurator { ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(zk.getData(appPath, ConfigCurator.META_ZK_PATH)); assertThat(metaData.getApplicationName(), is("appName")); assertTrue(metaData.getCheckSum().length() > 0); + assertTrue(metaData.isInternalRedeploy()); assertThat(metaData.getDeployedByUser(), is("foo")); assertThat(metaData.getDeployPath(), is("/bar/baz")); assertThat(metaData.getDeployTimestamp(), is(1345l)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 157c36d7aef..9ba8adf0aa4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -6,7 +6,9 @@ import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileReferenceData; import org.junit.After; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -35,6 +37,9 @@ public class FileServerTest { created.add(dir); } + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test public void requireThatExistingFileCanBeFound() throws IOException { createCleanDir("123"); @@ -71,10 +76,12 @@ public class FileServerTest { } @Test - public void requireThatDifferentNumberOfConfigServersWork() { + public void requireThatDifferentNumberOfConfigServersWork() throws IOException { // Empty connection pool in tests etc. - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - FileServer fileServer = new FileServer(new ConfigserverConfig(builder)); + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()); + FileServer fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // Empty connection pool when only one server, no use in downloading from yourself @@ -84,7 +91,7 @@ public class FileServerTest { serverBuilder.port(123456); servers.add(serverBuilder); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // connection pool of size 1 when 2 servers @@ -93,10 +100,16 @@ public class FileServerTest { serverBuilder2.port(123456); servers.add(serverBuilder2); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); } + private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { + File fileReferencesDir = temporaryFolder.newFolder(); + configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); + return new FileServer(new ConfigserverConfig(configBuilder)); + } + private static class FileReceiver implements FileServer.Receiver { CompletableFuture<byte []> content; FileReceiver(CompletableFuture<byte []> content) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java index 9b82beb860e..5b166155a45 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java @@ -29,7 +29,7 @@ public class HttpConfigResponseTest { // TODO: Hope to be able to remove this mess soon. DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); InnerCNode targetDef = dParser.getTree(); - ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5"); + ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"); HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index b19d6e2e257..f3800d66330 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -55,7 +55,7 @@ public class HttpGetConfigHandlerTest { final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -82,7 +82,7 @@ public class HttpGetConfigHandlerTest { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 930ae98c9e9..eb5dc7a2abf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -191,7 +191,8 @@ public class SessionHandlerTest { public String applicationName; @Override - public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, TimeoutBudget timeoutBudget) { + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget) { createCalled = true; this.applicationName = applicationId.application().value(); if (doThrow) { @@ -208,7 +209,8 @@ public class SessionHandlerTest { } @Override - public LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, TimeoutBudget timeoutBudget) { + public LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { if (doThrow) { throw new RuntimeException("foo"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 5226ff38ce3..bc583c64206 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -60,7 +60,7 @@ public class HttpGetConfigHandlerTest { ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -75,7 +75,7 @@ public class HttpGetConfigHandlerTest { mockRequestHandler.responses.put(new ApplicationId.Builder() .tenant(tenant) .applicationName("myapplication").instanceName("myinstance").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -123,7 +123,7 @@ public class HttpGetConfigHandlerTest { ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index e14e59b9fe7..fba4e40000d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -50,7 +50,9 @@ import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; @@ -89,6 +91,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { private TenantRepository tenantRepository; private SessionActiveHandler handler; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setup() { remoteSessionRepo = new RemoteSessionRepo(tenantName); @@ -221,9 +226,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { return session; } - private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) { + private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) throws IOException { writeApplicationId(zkc, deployData.getApplicationName()); - TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName); + TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, deployData); localRepo.addSession(new LocalSession(tenantName, sessionId, new SessionTest.MockSessionPreparer(), new SessionContext(app, zkc, new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)), @@ -280,7 +285,13 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ActivateRequest(long sessionId, long previousSessionId, Session.Status initialStatus, String subPath, Clock clock) { this.sessionId = sessionId; this.initialStatus = initialStatus; - this.deployData = new DeployData("foo", "bar", appName, 0l, sessionId, previousSessionId); + this.deployData = new DeployData("foo", + "bar", + appName, + 0l, + false, + sessionId, + previousSessionId); this.subPath = subPath; this.clock = clock; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java new file mode 100644 index 00000000000..b92feffbb55 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.time.Clock; + +class MaintainerTester { + + private final Curator curator; + private final TenantRepository tenantRepository; + private final ApplicationRepository applicationRepository; + + MaintainerTester() { + curator = new MockCurator(); + GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); + tenantRepository = new TenantRepository(componentRegistry, false); + applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + } + + Curator curator() { return curator; } + TenantRepository tenantRepository() { return tenantRepository; } + + ApplicationRepository applicationRepository() { return applicationRepository;} + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java new file mode 100644 index 00000000000..63ee9dfe3d9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -0,0 +1,50 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.Test; + +import java.io.File; +import java.time.Duration; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class TenantsMaintainerTest { + + @Test + public void deleteTenantWithNoApplications() { + MaintainerTester tester = new MaintainerTester(); + TenantRepository tenantRepository = tester.tenantRepository(); + ApplicationRepository applicationRepository = tester.applicationRepository(); + + TenantName shouldBeDeleted = TenantName.from("to-be-deleted"); + TenantName shouldNotBeDeleted = TenantName.from("should-not-be-deleted"); + + tenantRepository.addTenant(shouldBeDeleted); + tenantRepository.addTenant(shouldNotBeDeleted); + tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); + applicationRepository.deploy(new File("src/test/apps/app"), + new PrepareParams.Builder() + .applicationId(ApplicationId.from(shouldNotBeDeleted, ApplicationName.from("foo"), InstanceName.defaultName())) + .build()); + assertNotNull(tenantRepository.getTenant(shouldBeDeleted)); + assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); + + new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1)).run(); + + // One tenant should now have been deleted + assertNull(tenantRepository.getTenant(shouldBeDeleted)); + assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); + + // System tenants should not be deleted + assertNotNull(tenantRepository.getTenant(TenantName.defaultName())); + assertNotNull(tenantRepository.getTenant(TenantRepository.HOSTED_VESPA_TENANT)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java new file mode 100644 index 00000000000..ceb9b7129b4 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ZooKeeperDataMaintainerTest { + + @Test + public void deleteOldData() { + MaintainerTester tester = new MaintainerTester(); + Curator curator = tester.curator(); + + curator.create(Path.fromString("/foo")); + curator.create(Path.fromString("/vespa/bar")); + curator.create(Path.fromString("/vespa/filedistribution")); + curator.create(Path.fromString("/vespa/config")); + + new ZooKeeperDataMaintainer(tester.applicationRepository(), curator, Duration.ofDays(1)).run(); + + assertTrue(curator.exists(Path.fromString("/foo"))); + assertTrue(curator.exists(Path.fromString("/vespa"))); + assertFalse(curator.exists(Path.fromString("/vespa/filedistribution"))); + assertFalse(curator.exists(Path.fromString("/vespa/config"))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java index 9193c2409c7..7c1d5fa8dbc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactoryTest.java @@ -13,38 +13,38 @@ import org.junit.Test; import java.io.StringReader; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** - * @author lulf - * @since 5.19 + * @author Ulf Lilleengen */ public class ConfigResponseFactoryTest { - private InnerCNode def; + private InnerCNode def; @Before public void setup() { - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), + new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); def = dParser.getTree(); } @Test public void testUncompressedFacory() { UncompressedConfigResponseFactory responseFactory = new UncompressedConfigResponseFactory(); - ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3); - assertThat(response.getCompressionInfo().getCompressionType(), is(CompressionType.UNCOMPRESSED)); - assertThat(response.getGeneration(), is(3l)); - assertThat(response.getPayload().getByteLength(), is(2)); + ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3, false); + assertEquals(CompressionType.UNCOMPRESSED, response.getCompressionInfo().getCompressionType()); + assertEquals(3L,response.getGeneration()); + assertEquals(2, response.getPayload().getByteLength()); } @Test public void testLZ4CompressedFacory() { LZ4ConfigResponseFactory responseFactory = new LZ4ConfigResponseFactory(); - ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3); - assertThat(response.getCompressionInfo().getCompressionType(), is(CompressionType.LZ4)); - assertThat(response.getGeneration(), is(3l)); - assertThat(response.getPayload().getByteLength(), is(3)); + ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3, false); + assertEquals(CompressionType.LZ4, response.getCompressionInfo().getCompressionType()); + assertEquals(3L, response.getGeneration()); + assertEquals(3, response.getPayload().getByteLength()); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java index 0126a9e2f29..9455798c4ea 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java @@ -12,8 +12,11 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.GetConfigContext; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -28,10 +31,13 @@ import static org.junit.Assert.assertTrue; */ public class DelayedConfigResponseTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testDelayedConfigResponses() { + public void testDelayedConfigResponses() throws IOException { - MockRpc rpc = new MockRpc(13337); + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 1000000, "bar"); @@ -50,9 +56,9 @@ public class DelayedConfigResponseTest { } @Test - public void testDelayResponseRemove() { + public void testDelayResponseRemove() throws IOException { GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); - MockRpc rpc = new MockRpc(13337); + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); assertThat(responses.size(), is(1)); @@ -61,8 +67,8 @@ public class DelayedConfigResponseTest { } @Test - public void testDelayedConfigResponse() { - MockRpc rpc = new MockRpc(13337); + public void testDelayedConfigResponse() throws IOException { + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0")); @@ -72,7 +78,7 @@ public class DelayedConfigResponseTest { assertThat(rpc.latestRequest, is(req)); } - public JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { + private JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { Request request = JRTClientConfigRequestV3. createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()), "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java index 578224833a1..1a4d04d0323 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java @@ -16,10 +16,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.rpc.GetConfigProcessor; -import com.yahoo.vespa.config.server.rpc.MockRpc; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import static org.junit.Assert.assertFalse; import java.io.IOException; @@ -39,9 +40,12 @@ import static org.junit.Assert.assertTrue; */ public class GetConfigProcessorTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testSentinelConfig() { - MockRpc rpc = new MockRpc(13337, false); + public void testSentinelConfig() throws IOException { + MockRpc rpc = new MockRpc(13337, false, temporaryFolder.newFolder()); rpc.response = new MockConfigResponse("foo"); // should be a sentinel config, but it does not matter for this test // one tenant, which has host1 assigned @@ -104,6 +108,9 @@ public class GetConfigProcessorTest { } @Override + public boolean isInternalRedeploy() { return false; } + + @Override public String getConfigMd5() { return "mymd5"; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java index 62ff13093ea..ebdee8f58e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java @@ -15,15 +15,15 @@ import java.util.*; /** * Test utility class - * @author lulf - * @since 5.25 + * + * @author Ulf Lilleengen */ public class MockRequestHandler implements RequestHandler, ReloadHandler, TenantHandlerProvider { volatile boolean throwException = false; private Set<ConfigKey<?>> allConfigs = new HashSet<>(); public volatile ConfigResponse responseConfig = null; // for some v1 mocking - public Map<ApplicationId, ConfigResponse> responses = new LinkedHashMap<>(); // for v2 mocking + public Map<ApplicationId, ConfigResponse> responses = new LinkedHashMap<>(); // for v3 mocking private final boolean pretendToHaveLoadedAnyApplication; public MockRequestHandler() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 5a9735f774a..b4de201bd0b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -14,6 +13,7 @@ import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; +import java.io.File; import java.util.Optional; import java.util.concurrent.CompletionService; @@ -36,20 +36,20 @@ public class MockRpc extends RpcServer { public volatile JRTServerConfigRequest latestRequest = null; - public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) { + public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication, File tempDir) { super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir)); if (createDefaultTenant) { onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); } } - public MockRpc(int port, boolean createDefaultTenant) { - this(port, createDefaultTenant, true); + public MockRpc(int port, boolean createDefaultTenant, File tempDir) { + this(port, createDefaultTenant, true, tempDir); } - public MockRpc(int port) { - this(port, true); + public MockRpc(int port, File tempDir) { + this(port, true, tempDir); } /** Reset fields used to assert on the calls made to this */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index ef742ae3d38..9807045e122 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -32,8 +32,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class RpcServerTest extends TestWithRpc { @@ -48,7 +47,7 @@ public class RpcServerTest extends TestWithRpc { testEmptyConfigHostedVespa(); } - private void testEmptyConfigHostedVespa() throws InterruptedException { + private void testEmptyConfigHostedVespa() throws InterruptedException, IOException { rpcServer.onTenantDelete(TenantName.defaultName()); rpcServer.onTenantsLoaded(); JRTClientConfigRequest clientReq = createSimpleRequest(); @@ -70,10 +69,15 @@ public class RpcServerTest extends TestWithRpc { return clientReq; } - private void testEnabled() throws IOException, SAXException { generationCounter.increment(); - Application app = new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 2l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), ApplicationId.defaultId()); + Application app = new Application(new VespaModel(MockApplicationPackage.createEmpty()), + new ServerCache(), + 2L, + false, + Version.fromIntValues(1, 2, 3), + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()); ApplicationSet appSet = ApplicationSet.fromSingle(app); rpcServer.configActivated(TenantName.defaultName(), appSet); ConfigKey<?> key = new ConfigKey<>(LbServicesConfig.class, "*"); @@ -95,12 +99,17 @@ public class RpcServerTest extends TestWithRpc { public void testGetConfig() { ((MockRequestHandler)tenantProvider.getRequestHandler()).throwException = false; ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, "brim"); - ((MockRequestHandler)tenantProvider.getRequestHandler()).responses.put(ApplicationId.defaultId(), createResponse()); - JRTClientConfigRequest req = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, SimpletypesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); + ((MockRequestHandler)tenantProvider.getRequestHandler()).responses.put(ApplicationId.defaultId(), createResponse(true)); + JRTClientConfigRequest req = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, SimpletypesConfig.CONFIG_DEF_MD5), + 120_000, + Trace.createDummy(), + CompressionType.UNCOMPRESSED, + Optional.empty()); assertTrue(req.validateParameters()); performRequest(req.getRequest()); assertThat(req.errorCode(), is(0)); assertTrue(req.validateResponse()); + assertTrue(req.responseIsInternalRedeploy()); ConfigPayload payload = ConfigPayload.fromUtf8Array(req.getNewPayload().getData()); assertNotNull(payload); SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); @@ -109,13 +118,19 @@ public class RpcServerTest extends TestWithRpc { assertThat(config.intval(), is(123)); } - public ConfigResponse createResponse() { + public ConfigResponse createResponse(boolean internalRedeploy) { SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); builder.intval(123); SimpletypesConfig responseConfig = new SimpletypesConfig(builder); ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); - InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))).getTree(); - return SlimeConfigResponse.fromConfigPayload(responsePayload, targetDef, 3l, ConfigUtils.getMd5(responsePayload)); + InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, + new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))) + .getTree(); + return SlimeConfigResponse.fromConfigPayload(responsePayload, + targetDef, + 3L, + internalRedeploy, + ConfigUtils.getMd5(responsePayload)); } public void testPrintStatistics() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index e022b622fb0..845e7c0f914 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; @@ -20,7 +19,10 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import org.junit.After; import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -36,8 +38,7 @@ import static org.junit.Assert.assertTrue; /** * Test running rpc server. * - * @author lulf - * @since 5.17 + * @author Ulf Lilleengen */ // TODO: Make this a Tester instead of a superclass public class TestWithRpc { @@ -56,8 +57,11 @@ public class TestWithRpc { private List<Integer> allocatedPorts; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRpc() throws InterruptedException { + public void setupRpc() throws InterruptedException, IOException { allocatedPorts = new ArrayList<>(); port = allocatePort(); spec = createSpec(port); @@ -80,7 +84,7 @@ public class TestWithRpc { return port; } - protected void createAndStartRpcServer(boolean hostedVespa) { + protected void createAndStartRpcServer(boolean hostedVespa) throws IOException { ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder() .rpcport(port) @@ -94,7 +98,7 @@ public class TestWithRpc { emptyNodeFlavors(), generationCounter)), Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath())); + hostLivenessTracker, new FileServer(temporaryFolder.newFolder())); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); t.start(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 73caf770512..987dd8a6c4d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -14,7 +14,9 @@ import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.time.Duration; @@ -34,6 +36,9 @@ public class LocalSessionRepoTest extends TestWithCurator { private ManualClock clock; private static final TenantName tenantName = TenantName.defaultName(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setupSessions() throws Exception { setupSessions(tenantName, true); @@ -41,7 +46,7 @@ public class LocalSessionRepoTest extends TestWithCurator { private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName); + TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); if (createInitialSessions) { IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "1")); IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "2")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index bd5cfdf3a0e..316c439a3cd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.*; import com.yahoo.path.Path; @@ -29,8 +30,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class LocalSessionTest { @@ -41,7 +41,7 @@ public class LocalSessionTest { private SuperModelGenerationCounter superModelGenerationCounter; @Before - public void setupTest() throws Exception { + public void setupTest() { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); superModelGenerationCounter = new SuperModelGenerationCounter(curator); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java index 9d8b7c5cc00..2c05017d449 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider; @@ -11,7 +12,12 @@ import java.io.File; */ public class MockFileDistributionFactory extends FileDistributionFactory { - public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider(); + public final MockFileDistributionProvider mockFileDistributionProvider; + + public MockFileDistributionFactory(File fileReferencesDir) { + super(new ConfigserverConfig(new ConfigserverConfig.Builder())); + mockFileDistributionProvider = new MockFileDistributionProvider(fileReferencesDir); + } @Override public FileDistributionProvider createProvider(File applicationFile) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java index 531a2e3745b..0ca487cfb67 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java @@ -31,8 +31,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionFactoryTest extends TestWithTenant { private SessionFactory factory; @@ -64,10 +63,13 @@ public class SessionFactoryTest extends TestWithTenant { public void require_that_session_can_be_created_from_existing() throws IOException { LocalSession session = getLocalSession(); assertNotNull(session); - assertThat(session.getSessionId(), is(2l)); - LocalSession session2 = factory.createSessionFromExisting(session, new BaseDeployLogger(), TimeoutBudgetTest.day()); + assertThat(session.getSessionId(), is(2L)); + LocalSession session2 = factory.createSessionFromExisting(session, + new BaseDeployLogger(), + false, + TimeoutBudgetTest.day()); assertNotNull(session2); - assertThat(session2.getSessionId(), is(3l)); + assertThat(session2.getSessionId(), is(3L)); } @Test(expected = RuntimeException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java index 77505006b77..4d01f8a609d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java @@ -7,8 +7,7 @@ import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; /** - * @author lulf - * @since 5. + * @author Ulf Lilleengen */ public class MockTenantProvider implements TenantHandlerProvider { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 42c2d8db968..f47ed69ad14 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -65,8 +65,13 @@ public class TenantRepositoryTest extends TestWithCurator { @Test public void testListenersAdded() throws IOException, SAXException { tenantRepository.getTenant(tenant1).getReloadHandler().reloadConfig(ApplicationSet.fromSingle( - new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 4l, - Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), ApplicationId.defaultId()))); + new Application(new VespaModel(MockApplicationPackage.createEmpty()), + new ServerCache(), + 4L, + false, + Version.fromIntValues(1, 2, 3), + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()))); assertThat(listener.reloaded.get(), is(1)); } @@ -128,32 +133,6 @@ public class TenantRepositoryTest extends TestWithCurator { } @Test - public void testTenantsChanged() { - tenantRepository.close(); // Close the repo created in setup() - TenantRepository tenantRepository = new TenantRepository(globalComponentRegistry); - tenantRepository.addTenant(tenant2); - tenantRepository.createTenants(); - Set<TenantName> allTenants = tenantRepository.getAllTenantNames(); - assertTrue(allTenants.contains(tenant2)); - tenantRepository.deleteTenant(tenant1); - tenantRepository.deleteTenant(tenant2); - tenantRepository.createTenants(); - allTenants = tenantRepository.getAllTenantNames(); - assertFalse(allTenants.contains(tenant1)); - assertFalse(allTenants.contains(tenant2)); - TenantName foo = TenantName.from("foo"); - TenantName bar = TenantName.from("bar"); - tenantRepository.addTenant(tenant2); - tenantRepository.addTenant(foo); - tenantRepository.addTenant(bar); - tenantRepository.createTenants(); - allTenants = tenantRepository.getAllTenantNames(); - assertTrue(allTenants.contains(tenant2)); - assertTrue(allTenants.contains(foo)); - assertTrue(allTenants.contains(bar)); - } - - @Test public void testTenantWatching() throws Exception { TenantName newTenant = TenantName.from("newTenant"); List<TenantName> expectedTenants = Arrays.asList(TenantName.defaultName(), newTenant); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index ef320f0f084..cecbab2d9ec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.ConfigInstance; import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationName; @@ -54,8 +56,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class TenantRequestHandlerTest extends TestWithCurator { @@ -70,9 +71,13 @@ public class TenantRequestHandlerTest extends TestWithCurator { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + private ApplicationId defaultApp() { + return new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); + } + @Before - public void setUp() throws IOException, SAXException { - feedApp(app1, 1); + public void setUp() throws IOException { + feedApp(app1, 1, defaultApp(), false); Metrics sh = Metrics.createTestMetrics(); List<ReloadListener> listeners = new ArrayList<>(); listeners.add(listener); @@ -80,11 +85,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { componentRegistry = new TestComponentRegistry.Builder().curator(curator).modelFactoryRegistry(createRegistry()).build(); } - private void feedApp(File appDir, long sessionId) throws IOException { - feedApp(appDir, sessionId, new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build()); - } - - private void feedApp(File appDir, long sessionId, ApplicationId appId) throws IOException { + private void feedApp(File appDir, long sessionId, ApplicationId appId, boolean internalRedeploy) throws IOException { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId)), new TestConfigDefinitionRepo(), @@ -93,7 +94,17 @@ public class TenantRequestHandlerTest extends TestWithCurator { File app = tempFolder.newFolder(); IOUtils.copyDirectory(appDir, app); ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); + DeployData deployData = new DeployData("user", + appDir.toString(), + appId.application().toString(), + 0L, + internalRedeploy, + 0L, + 0L); + ApplicationPackage appPackage = FilesApplicationPackage.fromFileWithDeployData(appDir, deployData); + deployer.deploy(appPackage, + Collections.singletonMap(vespaVersion, new MockFileRegistry()), + AllocatedHosts.withHosts(Collections.emptySet())); } private ApplicationSet reloadConfig(long id, Clock clock) { @@ -115,12 +126,21 @@ public class TenantRequestHandlerTest extends TestWithCurator { new TestModelFactory(Version.fromIntValues(3, 2, 1)))); } - public <T extends ConfigInstance> T resolve(Class<T> clazz, TenantRequestHandler tenantRequestHandler, String configId) { - return resolve(clazz, tenantRequestHandler, new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), vespaVersion, configId); + public <T extends ConfigInstance> T resolve(Class<T> clazz, + TenantRequestHandler tenantRequestHandler, + ApplicationId appId, + Version vespaVersion, + String configId) { + ConfigResponse response = getConfigResponse(clazz, tenantRequestHandler, appId, vespaVersion, configId); + return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); } - public <T extends ConfigInstance> T resolve(final Class<T> clazz, TenantRequestHandler tenantRequestHandler, ApplicationId appId, Version vespaVersion, final String configId) { - ConfigResponse response = tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { + public <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, + TenantRequestHandler tenantRequestHandler, + ApplicationId appId, + Version vespaVersion, + String configId) { + return tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { @Override public ConfigKey<T> getConfigKey() { return new ConfigKey<T>(clazz, configId); @@ -141,23 +161,24 @@ public class TenantRequestHandlerTest extends TestWithCurator { return false; } }, Optional.empty()); - return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); } @Test - public void testReloadConfig() throws IOException, SAXException { + public void testReloadConfig() throws IOException { Clock clock = Clock.systemUTC(); ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); server.reloadConfig(reloadConfig(1, clock)); assertThat(listener.reloaded.get(), is(1)); // Using only payload list for this simple test - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, ""); + SimpletypesConfig config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); assertThat(config.intval(), is(1337)); assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1l)); - server.reloadConfig(reloadConfig(1l, clock)); - config = resolve(SimpletypesConfig.class, server, ""); + server.reloadConfig(reloadConfig(1L, clock)); + ConfigResponse configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); + assertFalse(configResponse.isInternalRedeploy()); + config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); assertThat(config.intval(), is(1337)); assertThat(listener.reloaded.get(), is(2)); assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1l)); @@ -165,9 +186,11 @@ public class TenantRequestHandlerTest extends TestWithCurator { assertThat(server.resolveApplicationId("mytesthost"), is(applicationId)); listener.reloaded.set(0); - feedApp(app2, 2); - server.reloadConfig(reloadConfig(2l, clock)); - config = resolve(SimpletypesConfig.class, server, ""); + feedApp(app2, 2, defaultApp(), true); + server.reloadConfig(reloadConfig(2L, clock)); + configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); + assertTrue(configResponse.isInternalRedeploy()); + config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion,""); assertThat(config.intval(), is(1330)); assertThat(listener.reloaded.get(), is(1)); assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(2l)); @@ -183,7 +206,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { @Test public void testResolveForAppId() { - long id = 1l; + long id = 1L; SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id)), new TestConfigDefinitionRepo(), @@ -199,7 +222,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { } @Test - public void testResolveMultipleApps() throws IOException, SAXException { + public void testResolveMultipleApps() throws IOException { ApplicationId appId1 = new ApplicationId.Builder() .tenant(tenant) .applicationName("myapp1").instanceName("myinst1").build(); @@ -228,7 +251,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { } private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { - feedApp(appDir, sessionId, appId); + feedApp(appDir, sessionId, appId, false); SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); zkc.writeApplicationId(appId); RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc, Clock.systemUTC()); @@ -263,7 +286,8 @@ public class TenantRequestHandlerTest extends TestWithCurator { public void testHasApplication() throws IOException, SAXException { assertdefaultAppNotFound(); server.reloadConfig(reloadConfig(1l, Clock.systemUTC())); - assertTrue(server.hasApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), Optional.of(vespaVersion))); + assertTrue(server.hasApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), + Optional.of(vespaVersion))); } private void assertdefaultAppNotFound() { @@ -286,18 +310,17 @@ public class TenantRequestHandlerTest extends TestWithCurator { @Test public void testListConfigs() throws IOException, SAXException { assertdefaultAppNotFound(); - /*assertTrue(server.allConfigIds(ApplicationId.defaultId()).isEmpty()); - assertTrue(server.allConfigsProduced(ApplicationId.defaultId()).isEmpty()); - assertTrue(server.listConfigs(ApplicationId.defaultId(), false).isEmpty()); - assertTrue(server.listConfigs(ApplicationId.defaultId(), true).isEmpty());*/ VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); - server.reloadConfig(ApplicationSet.fromSingle(new Application(model, new ServerCache(), 1, vespaVersion, MetricUpdater.createTestUpdater(), ApplicationId.defaultId()))); + server.reloadConfig(ApplicationSet.fromSingle(new Application(model, + new ServerCache(), + 1, + false, + vespaVersion, + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()))); Set<ConfigKey<?>> configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); - //for (ConfigKey<?> ck : configNames) { - // assertTrue(!"".equals(ck.getConfigId())); - //} configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); System.out.println(configNames); diff --git a/container-core/src/main/java/com/yahoo/container/Server.java b/container-core/src/main/java/com/yahoo/container/Server.java index 207050a8d88..2eec2e36c47 100644 --- a/container-core/src/main/java/com/yahoo/container/Server.java +++ b/container-core/src/main/java/com/yahoo/container/Server.java @@ -19,7 +19,7 @@ public class Server { private ConfigSubscriber subscriber = new ConfigSubscriber(); /** The OSGi container instance of this server */ - private Container container=Container.get(); + private Container container = Container.get(); /** A short string which is different for all the qrserver instances on a given node. */ private String localServerDiscriminator = "qrserver.0"; @@ -48,7 +48,6 @@ public class Server { return instance; } - private void initRpcServer(Rpc rpcConfig) { if (rpcConfig.enabled()) { ContainerRpcAdaptor rpcAdaptor = container.getRpcAdaptor(); @@ -57,8 +56,7 @@ public class Server { } } - /** Ugly hack, see Container.resetInstance - **/ + /** Ugly hack, see Container.resetInstance */ static void resetInstance() { instance = new Server(); } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 49cacb3a09b..cb4a21137a2 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -86,7 +86,7 @@ public class HandlersConfigurerDi { container = new Container(subscriberFactory, configId, deconstructor, osgiWrapper); try { - runOnceAndEnsureRegistryHackRun(discInjector); + getNewComponentGraph(discInjector, false); } catch (InterruptedException e) { throw new RuntimeException("Interrupted while setting up handlers for the first time."); } @@ -138,11 +138,13 @@ public class HandlersConfigurerDi { } } - public void runOnceAndEnsureRegistryHackRun(Injector discInjector) throws InterruptedException { - currentGraph = container.runOnce(currentGraph, createFallbackInjector(vespaContainer, discInjector)); + /** + * Wait for new config to arrive and produce the new graph + */ + public void getNewComponentGraph(Injector discInjector, boolean restartOnRedeploy) throws InterruptedException { + currentGraph = container.getNewComponentGraph(currentGraph, createFallbackInjector(vespaContainer, discInjector), restartOnRedeploy); - RegistriesHack registriesHack = currentGraph.getInstance(RegistriesHack.class); - assert (registriesHack != null); + assert (currentGraph.getInstance(RegistriesHack.class) != null); // TODO: Remove, seems quite pointless? } @SuppressWarnings("deprecation") diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 89c73e19fe3..afbf163500f 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -112,7 +112,7 @@ public class HandlersConfigurerTestWrapper { public void reloadConfig() { configurer.reloadConfig(++lastGeneration); try { - configurer.runOnceAndEnsureRegistryHackRun(Guice.createInjector()); + configurer.getNewComponentGraph(Guice.createInjector(), false); } catch (InterruptedException e) { throw new RuntimeException(e); } diff --git a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java index e4ec09e3948..31bceca9337 100644 --- a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java +++ b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.logging.Logger; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; @@ -37,6 +38,7 @@ import static java.util.stream.Collectors.toSet; * @author bjorncs */ public class FilterChainRepository extends AbstractComponent { + private static final Logger log = Logger.getLogger(FilterChainRepository.class.getName()); private final ComponentRegistry<Object> filterAndChains; @@ -77,12 +79,23 @@ public class FilterChainRepository extends AbstractComponent { ChainRegistry<FilterWrapper> chainRegistry = new ChainRegistry<>(); ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig); ChainsConfigurer.prepareChainRegistry(chainRegistry, chainsModel, allFiltersWrapped(filters)); + removeEmptyChains(chainRegistry); chainRegistry.freeze(); return chainRegistry; } + private static void removeEmptyChains(ChainRegistry<FilterWrapper> chainRegistry) { + chainRegistry.allComponents().stream() + .filter(chain -> chain.components().isEmpty()) + .map(Chain::getId) + .peek(id -> log.warning("Removing empty filter chain: " + id)) + .forEach(chainRegistry::unregister); + } + @SuppressWarnings("unchecked") private static Object toJDiscChain(Chain<FilterWrapper> chain) { + if (chain.components().isEmpty()) + throw new IllegalArgumentException("Empty filter chain: " + chain.getId()); checkFilterTypesCompatible(chain); List<?> jdiscFilters = chain.components().stream() .map(filterWrapper -> filterWrapper.filter) @@ -98,7 +111,6 @@ public class FilterChainRepository extends AbstractComponent { } private static List<?> wrapSecurityFilters(List<?> filters) { - if (filters.isEmpty()) return emptyList(); List<Object> aggregatedSecurityFilters = new ArrayList<>(); List<Object> wrappedFilters = new ArrayList<>(); for (Object filter : filters) { diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 1f94372fc14..b4af6800768 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -464,7 +464,7 @@ <guava.version>18.0</guava.version> <guice.version>3.0</guice.version> <jaxb.version>2.2.7</jaxb.version> - <jetty.version>9.4.9.v20180320</jetty.version> + <jetty.version>9.4.10.v20180503</jetty.version> <scala.version>2.11.12</scala.version> <!-- When updating this, the scala.major-version in parent must also be updated! --> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-di/src/main/java/com/yahoo/container/di/config/Subscriber.java b/container-di/src/main/java/com/yahoo/container/di/config/Subscriber.java index 9fd30f888b9..0feab7779ad 100644 --- a/container-di/src/main/java/com/yahoo/container/di/config/Subscriber.java +++ b/container-di/src/main/java/com/yahoo/container/di/config/Subscriber.java @@ -14,6 +14,7 @@ public interface Subscriber { long waitNextGeneration(); long generation(); + boolean internalRedeploy(); boolean configChanged(); Map<ConfigKey<ConfigInstance>, ConfigInstance> config(); diff --git a/container-di/src/main/scala/com/yahoo/container/di/CloudSubscriberFactory.scala b/container-di/src/main/scala/com/yahoo/container/di/CloudSubscriberFactory.scala index afef3e96821..0f3fab93e80 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/CloudSubscriberFactory.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/CloudSubscriberFactory.scala @@ -16,9 +16,8 @@ import scala.language.existentials /** * @author Tony Vaagenes */ +class CloudSubscriberFactory(configSource: ConfigSource) extends SubscriberFactory { -class CloudSubscriberFactory(configSource: ConfigSource) extends SubscriberFactory -{ private var testGeneration: Option[Long] = None private val activeSubscribers = new java.util.WeakHashMap[CloudSubscriber, Int]() @@ -50,9 +49,12 @@ object CloudSubscriberFactory { private val handles: Map[ConfigKeyT, ConfigHandle[_ <: ConfigInstance]] = keys.map(subscribe).toMap - //if waitNextGeneration has not yet been called, -1 should be returned + // if waitNextGeneration has not yet been called, -1 should be returned var generation: Long = -1 + // True if this reconfiguration was caused by a system-internal redeploy, not an external application change + var internalRedeploy: Boolean = false + private def subscribe(key: ConfigKeyT) = (key, subscriber.subscribe(key.getConfigClass, key.getConfigId)) override def configChanged = handles.values.exists(_.isChanged) @@ -79,13 +81,14 @@ object CloudSubscriberFactory { case e: IllegalArgumentException => numExceptions += 1 log.log(Level.WARNING, "Got exception from the config system (please ignore the exception if you just removed " - + "a component from your application that used the mentioned config): ", e) + + "a component from your application that used the mentioned config): ", e) if (numExceptions >= 5) throw new IllegalArgumentException("Failed retrieving the next config generation.", e) } } generation = subscriber.getGeneration + internalRedeploy = subscriber.isInternalRedeploy generation } diff --git a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala index dc94d789f7b..aad9e17acb2 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/ConfigRetriever.scala @@ -27,27 +27,43 @@ final class ConfigRetriever(bootstrapKeys: Set[ConfigKeyT], private var componentSubscriber: Subscriber = subscribe(Set()) private var componentSubscriberKeys: Set[ConfigKeyT] = Set() - + /** Loop forever until we get config */ @tailrec - final def getConfigs(componentConfigKeys: Set[ConfigKeyT], leastGeneration: Long): ConfigSnapshot = { + final def getConfigs(componentConfigKeys: Set[ConfigKeyT], leastGeneration: Long, restartOnRedeploy: Boolean = false): ConfigSnapshot = { require(componentConfigKeys intersect bootstrapKeys isEmpty) log.log(DEBUG, "getConfigs: " + componentConfigKeys) setupComponentSubscriber(componentConfigKeys ++ bootstrapKeys) - getConfigsOptional(leastGeneration) match { + getConfigsOptional(leastGeneration, restartOnRedeploy) match { case Some(snapshot) => resetComponentSubscriberIfBootstrap(snapshot); snapshot - case None => getConfigs(componentConfigKeys, leastGeneration) + case None => getConfigs(componentConfigKeys, leastGeneration, restartOnRedeploy) + } + } + + + /** Try to get config just once */ + final def getConfigsOnce(componentConfigKeys: Set[ConfigKeyT], leastGeneration: Long, restartOnRedeploy: Boolean = false): Option[ConfigSnapshot] = { + require(componentConfigKeys intersect bootstrapKeys isEmpty) + log.log(DEBUG, "getConfigsOnce: " + componentConfigKeys) + + setupComponentSubscriber(componentConfigKeys ++ bootstrapKeys) + + getConfigsOptional(leastGeneration, restartOnRedeploy) match { + case Some(snapshot) => resetComponentSubscriberIfBootstrap(snapshot); Some(snapshot) + case None => None; } } - private def getConfigsOptional(leastGeneration: Long): Option[ConfigSnapshot] = { + private def getConfigsOptional(leastGeneration: Long, restartOnRedeploy: Boolean): Option[ConfigSnapshot] = { val newestComponentGeneration = componentSubscriber.waitNextGeneration() log.log(DEBUG, s"getConfigsOptional: new component generation: $newestComponentGeneration") // leastGeneration is only used to ensure newer generation when the previous generation was invalidated due to an exception if (newestComponentGeneration < leastGeneration) { None + } else if (restartOnRedeploy && ! componentSubscriber.internalRedeploy()) { // Don't reconfig - wait for restart + None } else if (bootstrapSubscriber.generation < newestComponentGeneration) { val newestBootstrapGeneration = bootstrapSubscriber.waitNextGeneration() log.log(DEBUG, s"getConfigsOptional: new bootstrap generation: ${bootstrapSubscriber.generation}") diff --git a/container-di/src/main/scala/com/yahoo/container/di/Container.scala b/container-di/src/main/scala/com/yahoo/container/di/Container.scala index 5ea6422ddad..2a185d41a6c 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/Container.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/Container.scala @@ -44,9 +44,9 @@ class Container( var leastGeneration = -1L @throws(classOf[InterruptedException]) - def runOnce( - oldGraph: ComponentGraph = new ComponentGraph, - fallbackInjector: GuiceInjector = Guice.createInjector()): ComponentGraph = { + def getNewComponentGraph(oldGraph: ComponentGraph = new ComponentGraph, + fallbackInjector: GuiceInjector = Guice.createInjector(), + restartOnRedeploy: Boolean = false): ComponentGraph = { def deconstructObsoleteComponents(oldGraph: ComponentGraph, newGraph: ComponentGraph) { val oldComponents = new IdentityHashMap[AnyRef, AnyRef]() @@ -56,7 +56,7 @@ class Container( } try { - val newGraph = createNewGraph(oldGraph, fallbackInjector) + val newGraph = getConfigAndCreateGraph(oldGraph, fallbackInjector, restartOnRedeploy) newGraph.reuseNodes(oldGraph) constructComponents(newGraph) deconstructObsoleteComponents(oldGraph, newGraph) @@ -113,10 +113,12 @@ class Container( } } - final def createNewGraph(graph: ComponentGraph = new ComponentGraph, - fallbackInjector: Injector): ComponentGraph = { + final def getConfigAndCreateGraph(graph: ComponentGraph = new ComponentGraph, + fallbackInjector: Injector, + restartOnRedeploy: Boolean): ComponentGraph = { + + val snapshot = configurer.getConfigs(graph.configKeys, leastGeneration, restartOnRedeploy) - val snapshot = configurer.getConfigs(graph.configKeys, leastGeneration) log.log(DEBUG, """createNewGraph: |graph.configKeys = %s @@ -138,9 +140,8 @@ class Container( |previous generation: %d""" .format(getBootstrapGeneration, getComponentsGeneration, previousConfigGeneration).stripMargin) installBundles(configs) - createNewGraph( - createComponentsGraph(configs, getBootstrapGeneration,fallbackInjector), - fallbackInjector) + getConfigAndCreateGraph( + createComponentsGraph(configs, getBootstrapGeneration,fallbackInjector), fallbackInjector, restartOnRedeploy) case ComponentsConfigs(configs) => log.log(DEBUG, """Got components configs, diff --git a/container-di/src/test/java/demo/ContainerTestBase.java b/container-di/src/test/java/demo/ContainerTestBase.java index 426033ea101..9c2415c3514 100644 --- a/container-di/src/test/java/demo/ContainerTestBase.java +++ b/container-di/src/test/java/demo/ContainerTestBase.java @@ -12,12 +12,9 @@ import com.yahoo.container.di.Osgi; import com.yahoo.container.di.componentgraph.core.ComponentGraph; import org.junit.Before; import org.osgi.framework.Bundle; -import scala.collection.*; -import scala.collection.immutable.*; import scala.collection.immutable.Set; import java.util.Collection; -import java.util.List; /** * @author tonytv @@ -62,7 +59,7 @@ public class ContainerTestBase extends ContainerTest { throw new UnsupportedOperationException("getBundle not supported."); } }); - componentGraph = container.runOnce(componentGraph, Guice.createInjector()); + componentGraph = container.getNewComponentGraph(componentGraph, Guice.createInjector(), false); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/container-di/src/test/scala/com/yahoo/container/di/ConfigRetrieverTest.scala b/container-di/src/test/scala/com/yahoo/container/di/ConfigRetrieverTest.scala index 93618f90e92..7f1d9a73a82 100644 --- a/container-di/src/test/scala/com/yahoo/container/di/ConfigRetrieverTest.scala +++ b/container-di/src/test/scala/com/yahoo/container/di/ConfigRetrieverTest.scala @@ -18,11 +18,13 @@ import scala.collection.JavaConverters._ * @author tonytv */ class ConfigRetrieverTest { + var dirConfigSource: DirConfigSource = null @Before def setup() { dirConfigSource = new DirConfigSource("ConfigRetrieverTest-") } + @After def cleanup() { dirConfigSource.cleanup() } @Test @@ -49,6 +51,22 @@ class ConfigRetrieverTest { } } + @Test + def require_no_reconfig_when_restart_on_redeploy() { + // TODO + writeConfigs() + val retriever = createConfigRetriever() + val bootstrapConfigs = retriever.getConfigs(Set(), 0) + + val testConfigKey = new ConfigKey(classOf[TestConfig], dirConfigSource.configId) + val componentsConfigs = retriever.getConfigsOnce(Set(testConfigKey), 0, true) + + componentsConfigs match { + case Some(snapshot) => fail("Expected no configs") + case _ => // ok + } + } + @Test(expected = classOf[IllegalArgumentException]) @Ignore def require_exception_upon_modified_components_keys_without_bootstrap() { diff --git a/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala b/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala index 8e7fbde3f5e..9f07acc7dc9 100644 --- a/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala +++ b/container-di/src/test/scala/com/yahoo/container/di/ContainerTest.scala @@ -44,7 +44,7 @@ class ContainerTest { val container = newContainer(dirConfigSource) - val component = createComponentTakingConfig(container.runOnce()) + val component = createComponentTakingConfig(container.getNewComponentGraph()) assertThat(component.config.stringVal(), is("myString")) container.shutdownConfigurer() @@ -57,7 +57,7 @@ class ContainerTest { val container = newContainer(dirConfigSource) - val componentGraph = container.runOnce() + val componentGraph = container.getNewComponentGraph() val component = createComponentTakingConfig(componentGraph) assertThat(component.config.stringVal(), is("original")) @@ -66,7 +66,7 @@ class ContainerTest { dirConfigSource.writeConfig("test", """stringVal "reconfigured" """) container.reloadConfig(2) - val newComponentGraph = container.runOnce(componentGraph) + val newComponentGraph = container.getNewComponentGraph(componentGraph) val component2 = createComponentTakingConfig(newComponentGraph) assertThat(component2.config.stringVal(), is("reconfigured")) @@ -80,7 +80,7 @@ class ContainerTest { val container = newContainer(dirConfigSource) - val graph = container.runOnce() + val graph = container.getNewComponentGraph() val component = createComponentTakingConfig(graph) assertThat(component.getId.toString, is("id1")) @@ -89,7 +89,7 @@ class ContainerTest { ("id2", classOf[ComponentTakingConfig]))) container.reloadConfig(2) - val newGraph = container.runOnce(graph) + val newGraph = container.getNewComponentGraph(graph) assertThat(ComponentGraph.getNode(newGraph, "id1"), notNullValue(classOf[Node])) assertThat(ComponentGraph.getNode(newGraph, "id2"), notNullValue(classOf[Node])) @@ -107,12 +107,12 @@ class ContainerTest { val container = newContainer(dirConfigSource) - val oldGraph = container.runOnce() + val oldGraph = container.getNewComponentGraph() val componentToDestruct = oldGraph.getInstance(classOf[DestructableComponent]) writeBootstrapConfigs("id2", classOf[DestructableComponent]) container.reloadConfig(2) - container.runOnce(oldGraph) + container.getNewComponentGraph(oldGraph) assertTrue(componentToDestruct.deconstructed) } @@ -123,7 +123,7 @@ class ContainerTest { val container = newContainer(dirConfigSource) var currentGraph: ComponentGraph = null try { - currentGraph = container.runOnce() + currentGraph = container.getNewComponentGraph() fail("Expected to log and die.") } catch { case _: Throwable => fail("Expected to log and die") @@ -136,14 +136,14 @@ class ContainerTest { writeBootstrapConfigs(Array(simpleComponentEntry)) val container = newContainer(dirConfigSource) - var currentGraph = container.runOnce() + var currentGraph = container.getNewComponentGraph() val simpleComponent = currentGraph.getInstance(classOf[SimpleComponent]) writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionInConstructor]) container.reloadConfig(2) try { - currentGraph = container.runOnce(currentGraph) + currentGraph = container.getNewComponentGraph(currentGraph) fail("Expected exception") } catch { case _: ComponentConstructorException => // Expected, do nothing @@ -156,7 +156,7 @@ class ContainerTest { dirConfigSource.writeConfig("test", """stringVal "myString" """) writeBootstrapConfigs(Array(simpleComponentEntry, componentTakingConfigEntry)) container.reloadConfig(3) - currentGraph = container.runOnce(currentGraph) + currentGraph = container.getNewComponentGraph(currentGraph) assertEquals(3, currentGraph.generation) assertSame(simpleComponent, currentGraph.getInstance(classOf[SimpleComponent])) @@ -169,7 +169,7 @@ class ContainerTest { writeBootstrapConfigs(Array(simpleComponentEntry)) val container = newContainer(dirConfigSource) - var currentGraph = container.runOnce() + var currentGraph = container.getNewComponentGraph() val simpleComponent = currentGraph.getInstance(classOf[SimpleComponent]) @@ -177,7 +177,7 @@ class ContainerTest { dirConfigSource.writeConfig("test", """stringVal "myString" """) container.reloadConfig(2) try { - currentGraph = container.runOnce(currentGraph) + currentGraph = container.getNewComponentGraph(currentGraph) fail("Expected exception") } catch { case _: IllegalArgumentException => // Expected, do nothing @@ -192,20 +192,20 @@ class ContainerTest { writeBootstrapConfigs("myId", classOf[ComponentTakingConfig]) val container = newContainer(dirConfigSource) - var currentGraph = container.runOnce() + var currentGraph = container.getNewComponentGraph() writeBootstrapConfigs("thrower", classOf[ComponentThrowingExceptionForMissingConfig]) container.reloadConfig(2) try { - currentGraph = container.runOnce(currentGraph) + currentGraph = container.getNewComponentGraph(currentGraph) fail("expected exception") } catch { case e: Exception => } val newGraph = Future { - currentGraph = container.runOnce(currentGraph) + currentGraph = container.getNewComponentGraph(currentGraph) currentGraph } @@ -230,7 +230,7 @@ class ContainerTest { dirConfigSource.writeConfig("jersey-injection", """inject[0]" """) val container = newContainer(dirConfigSource) - val componentGraph = container.runOnce() + val componentGraph = container.getNewComponentGraph() val restApiContext = componentGraph.getInstance(clazz) assertNotNull(restApiContext) @@ -278,7 +278,7 @@ class ContainerTest { dirConfigSource.writeConfig("jersey-injection", injectionConfig) val container = newContainer(dirConfigSource) - val componentGraph = container.runOnce() + val componentGraph = container.getNewComponentGraph() val restApiContext = componentGraph.getInstance(restApiClass) } @@ -328,12 +328,12 @@ class ContainerTest { val container = newContainer(dirConfigSource, deconstructor) - val oldGraph = container.runOnce() + val oldGraph = container.getNewComponentGraph() val destructableEntity = oldGraph.getInstance(classOf[DestructableEntity]) writeBootstrapConfigs("id2", classOf[DestructableProvider]) container.reloadConfig(2) - container.runOnce(oldGraph) + container.getNewComponentGraph(oldGraph) assertTrue(destructableEntity.deconstructed) } diff --git a/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala b/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala index 5afa1bc418e..4f80b25a247 100644 --- a/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala +++ b/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala @@ -7,14 +7,12 @@ import java.util.Random import org.junit.rules.TemporaryFolder import com.yahoo.config.subscription.{ConfigSource, ConfigSourceSet} -// TODO: Make this a junit rule. Does not yet work. Look out for junit updates -// (@Rule def configSourceRule = dirConfigSource) - /** * @author tonytv * @author gjoranv */ class DirConfigSource(val testSourcePrefix: String) { + private val tempFolder = createTemporaryFolder() val configSource : ConfigSource = new ConfigSourceSet(testSourcePrefix + new Random().nextLong) @@ -32,9 +30,11 @@ class DirConfigSource(val testSourcePrefix: String) { def cleanup() { tempFolder.delete() } + } private object DirConfigSource { + def printFile(f: File, content: String) { var out: OutputStream = new FileOutputStream(f) out.write(content.getBytes("UTF-8")) @@ -46,4 +46,5 @@ private object DirConfigSource { folder.create() folder } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 1977b934253..932d31c0036 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -128,7 +128,7 @@ public final class ConfiguredApplication implements Application { configureComponents(builder.guiceModules().activate()); intitializeAndActivateContainer(builder); - if (! qrConfig.restartOnDeploy()) startReconfigurerThread(); + startReconfigurerThread(); portWatcher = new Thread(this::watchPortChange); portWatcher.setDaemon(true); portWatcher.start(); @@ -199,7 +199,9 @@ public final class ConfiguredApplication implements Application { while ( ! Thread.interrupted()) { try { ContainerBuilder builder = createBuilderWithGuiceBindings(); - configurer.runOnceAndEnsureRegistryHackRun(builder.guiceModules().activate()); + + // Block until new config arrives, and it should be applied + configurer.getNewComponentGraph(builder.guiceModules().activate(), qrConfig.restartOnDeploy()); intitializeAndActivateContainer(builder); } catch (ConfigInterruptedException | InterruptedException e) { break; diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index b899f690ee1..bdf395c1f0b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -166,8 +166,8 @@ public class IndexFacts { } private Index getIndexFromDocumentTypes(String indexName, List<String> documentTypes) { - if (indexName==null || indexName.isEmpty()) - indexName="default"; + if (indexName == null || indexName.isEmpty()) + indexName = "default"; return getIndexByCanonicNameFromDocumentTypes(indexName, documentTypes); } @@ -191,6 +191,13 @@ public class IndexFacts { return Index.nullIndex; } + private Collection<Index> getIndexes(String documentType) { + if ( ! isInitialized()) return Collections.emptyList(); + SearchDefinition sd = searchDefinitions.get(documentType); + if (sd == null) return Collections.emptyList(); + return sd.indices().values(); + } + /** Calls resolveDocumentTypes(query.getModel().getSources(), query.getModel().getRestrict()) */ private Set<String> resolveDocumentTypes(Query query) { // Assumption: Search definition name equals document name. @@ -421,6 +428,11 @@ public class IndexFacts { return IndexFacts.this.getIndexFromDocumentTypes(indexName, Collections.singletonList(documentType)); } + /** Returns all the indexes of a given search definition */ + public Collection<Index> getIndexes(String documentType) { + return IndexFacts.this.getIndexes(documentType); + } + /** * Returns the canonical form of the index name (Which may be the same as * the input). diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index 644cacfa322..47becde7b19 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -83,7 +83,7 @@ public class SearchDefinition { } /** Returns the indices of this as a map */ - public Map<String,Index> indices() { + public Map<String, Index> indices() { return indices; } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 91fbd1b2aca..b1399c6cc8d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -469,6 +469,7 @@ public class ClusterSearcher extends Searcher { } private List<Query> createQueries(Query query, Set<String> docTypes) { + query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once List<Query> retval = new ArrayList<>(docTypes.size()); if (docTypes.size() == 1) { query.getModel().setRestrict(docTypes.iterator().next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index a5f83021bee..7d68e7b6679 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -96,4 +96,7 @@ public abstract class DocsumField { */ public abstract Object convert(Inspector value); + /** Returns whether this is the string field type. */ + boolean isString() { return false; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 0b3ddf689d9..1160ea0a204 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -98,16 +98,18 @@ public class FastHit extends Hit { if (uri != null) return uri; // TODO: Remove on Vespa 7, this should be one of the last vestiges of URL field magic - if (fields().containsKey("uri")) { - // trigger decoding - Object o = getField("uri"); - setId(o.toString()); + Object uriField = getField("uri"); + if (uriField != null) { + setId(uriField.toString()); return super.getId(); } // Fallback to index:[source]/[partid]/[id] if (indexUri != null) return indexUri; - indexUri = new URI("index:" + getSource() + "/" + getPartId() + "/" + asHexString(getGlobalId())); + StringBuilder sb = new StringBuilder(64); + sb.append("index:").append(getSource()).append('/').append(getPartId()).append('/'); + asHexString(sb, getGlobalId()); + indexUri = new URI(sb.toString()); return indexUri; } @@ -226,6 +228,13 @@ public class FastHit extends Hit { } @Override + public void forEachFieldAsRaw(RawUtf8Consumer consumer) { + super.forEachField(consumer); + for (SummaryData summaryData : summaries) + summaryData.forEachFieldAsRaw(consumer); + } + + @Override public Map<String, Object> fields() { Map<String, Object> fields = new HashMap<>(); for (Iterator<Map.Entry<String, Object>> i = fieldIterator(); i.hasNext(); ) { @@ -348,7 +357,10 @@ public class FastHit extends Hit { /** @deprecated do not use */ @Deprecated // TODO: Make private on Vespa 7 public static String asHexString(GlobalId gid) { - StringBuilder sb = new StringBuilder(); + return asHexString(new StringBuilder(), gid).toString(); + } + + private static StringBuilder asHexString(StringBuilder sb, GlobalId gid) { byte[] rawGid = gid.getRawId(); for (byte b : rawGid) { String hex = Integer.toHexString(0xFF & b); @@ -357,7 +369,7 @@ public class FastHit extends Hit { } sb.append(hex); } - return sb.toString(); + return sb; } /** A set view of all the field names in this hit. Add/addAll is not supported but remove is. */ @@ -529,9 +541,28 @@ public class FastHit extends Hit { void forEachField(BiConsumer<String, Object> consumer) { data.traverse((ObjectTraverser)(name, value) -> { - Object convertedValue = type.convert(name, value); - if ( convertedValue != null && !shadowed(name) && !removed(name)) { - consumer.accept(name, convertedValue); + if (!shadowed(name) && !removed(name)) { + Object convertedValue = type.convert(name, value); + if (convertedValue != null) + consumer.accept(name, convertedValue); + } + }); + } + + void forEachFieldAsRaw(RawUtf8Consumer consumer) { + data.traverse((ObjectTraverser)(name, value) -> { + if (!shadowed(name) && !removed(name)) { + DocsumField fieldType = type.getField(name); + if (fieldType != null) { + if (fieldType.isString()) { + byte[] utf8Value = value.asUtf8(); + consumer.accept(name, utf8Value, 0, utf8Value.length); + } else { + Object convertedValue = fieldType.convert(value); + if (convertedValue != null) + consumer.accept(name, convertedValue); + } + } } }); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 54db3b083d7..30e2adab182 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -244,7 +244,7 @@ public class FastSearcher extends VespaBackEndSearcher { if (result.isFilled(summaryClass)) return; Query query = result.getQuery(); - traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); + traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); if (query.properties().getBoolean(dispatchSummaries, true) && ! summaryNeedsQuery(query) diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java index 0fa4b7ee342..408cbbbb62d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java @@ -31,4 +31,6 @@ public class StringField extends DocsumField { return value.asString(""); } + boolean isString() { return true; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java index e38c30c25ac..9d852c8822d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java +++ b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java @@ -4,8 +4,10 @@ package com.yahoo.prelude.logging; /** * Hollow compatibility class for com.yahoo.container.logging.AccessLogEntry. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class AccessLogEntry extends com.yahoo.container.logging.AccessLogEntry { public AccessLogEntry() { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index e8e0a07941e..0ba4133901a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -52,7 +52,7 @@ public abstract class Item implements Cloneable { WEIGHTEDSET(15), WEAK_AND(16), EXACT(17), - LEGACY_RISE_QUERY_NOT_USED_ANYMORE_BUT_DO_NOT_REUSE_FOR_A_WHILE(18), + SAME_ELEMENT(18), PURE_WEIGHTED_STRING(19), PURE_WEIGHTED_INTEGER(20), DOTPRODUCT(21), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java index 44555d19cc5..c3689805dd7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java @@ -205,8 +205,7 @@ public class PhraseItem extends CompositeIndexedItem { } /** Phrase items uses a empty heading instead of "PHRASE " */ - protected void appendHeadingString(StringBuilder buffer) { - } + protected void appendHeadingString(StringBuilder buffer) { } protected void appendBodyString(StringBuilder buffer) { appendIndexString(buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java new file mode 100644 index 00000000000..3d596cc7d34 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java @@ -0,0 +1,94 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import com.google.common.annotations.Beta; +import com.yahoo.protect.Validator; + +import java.nio.ByteBuffer; +import java.util.Iterator; + +/** + * This represents a query where all terms are required to match in the same element id. + * The primary usecase is to allow efficient search in arrays and maps of struct. + * The common path is the field name containing the struct. + * @author baldersheim + */ +@Beta +public class SameElementItem extends CompositeItem { + + private final String fieldName; + + public SameElementItem(String commonPath) { + Validator.ensureNonEmpty("Field name", commonPath); + this.fieldName = commonPath; + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + putString(fieldName, buffer); + } + + @Override + protected void appendHeadingString(StringBuilder buffer) { } + @Override + protected void appendBodyString(StringBuilder buffer) { + buffer.append(fieldName).append(':'); + buffer.append('{'); + for (Iterator<Item> i = getItemIterator(); i.hasNext();) { + TermItem term = (TermItem) i.next(); + buffer.append(extractSubFieldName(term)).append(':').append(term.getIndexedString()); + if (i.hasNext()) { + buffer.append(' '); + } + } + buffer.append('}'); + } + + @Override + protected void adding(Item item) { + Validator.ensureInstanceOf("Child item", item, TermItem.class); + TermItem asTerm = (TermItem) item; + Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName()); + Validator.ensureNonEmpty("Query term", asTerm.getIndexedString()); + Validator.ensure("Struct fieldname starts with '" + getFieldName() + ".'", + !asTerm.getIndexName().startsWith(fieldName+".") || (item.getParent() != null)); + super.adding(item); + } + + private void expandChild(Item item) { + item.setIndexName(fieldName + '.' + ((TermItem)item).getIndexName()); + } + @Override + public void addItem(int index, Item item) { + super.addItem(index, item); + expandChild(item); + } + + @Override + public void addItem(Item item) { + super.addItem(item); + expandChild(item); + } + + @Override + public Item setItem(int index, Item item) { + Item prev = super.setItem(index, item); + expandChild(item); + return prev; + } + + @Override + public ItemType getItemType() { + return ItemType.SAME_ELEMENT; + } + + @Override + public String getName() { + return getItemType().toString(); + } + public String getFieldName() { return fieldName; } + public String extractSubFieldName(TermItem full) { + return full.getIndexName().substring(getFieldName().length()+1); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 0384dfdca12..708c48f0954 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -42,11 +42,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { **/ public WeakAndItem(String index, int N) { this.N = N; - if (index == null) { - this.index = ""; - } else { - this.index = index; - } + this.index = (index == null) ? "" : index; } public WeakAndItem(int N) { this("", N); @@ -54,12 +50,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { /** Sets the index name of all subitems of this */ public void setIndexName(String index) { - String toSet; - if (index == null) { - toSet = ""; - } else { - toSet = index; - } + String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java index abf37c71b76..47e5651f64c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/CollapsePhraseSearcher.java @@ -15,9 +15,11 @@ import com.yahoo.search.searchchain.Execution; /** * Make single item phrases in query into single word items. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CollapsePhraseSearcher extends Searcher { + + @Override public Result search(Query query, Execution execution) { QueryTree tree = query.getModel().getQueryTree(); Item root = tree.getRoot(); @@ -35,7 +37,6 @@ public class CollapsePhraseSearcher extends Searcher { return execution.search(query); } - private Item simplifyPhrases(Item root) { if (root == null) { return root; @@ -64,4 +65,5 @@ public class CollapsePhraseSearcher extends Searcher { else return root; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java index 3d803b322ca..96e9fb30c24 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.Index.Attribute; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; @@ -22,14 +23,15 @@ import java.util.*; * Searcher to rewrite queries to achieve mixed recall between indices and * memory attributes. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ @After({PhaseNames.RAW_QUERY, PHRASE_REPLACEMENT}) @Before(PhaseNames.TRANSFORMED_QUERY) @Provides(IndexCombinatorSearcher.MIXED_RECALL_REWRITE) -// TODO: This is not necessary on Vespa 6, we should probably remove it from the default chain but keep it -// around until Vespa 6 to avoid breaking those who refer to it. +@Deprecated // TODO: Remove on Vespa 7 (not necessary any more) public class IndexCombinatorSearcher extends Searcher { + public static final String MIXED_RECALL_REWRITE = "MixedRecallRewrite"; private static class ArrayComparator implements Comparator<Attribute[]> { @@ -62,7 +64,7 @@ public class IndexCombinatorSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(Query query, Execution execution) { + public Result search(Query query, Execution execution) { Item root = query.getModel().getQueryTree().getRoot(); IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); String oldQuery = (query.getTraceLevel() >= 2) ? root.toString() : ""; @@ -175,22 +177,22 @@ public class IndexCombinatorSearcher extends Searcher { if (c instanceof NotItem) { c = rewriteNot((NotItem) c, session); return c; - } else if (c instanceof CompositeItem) { + } else { switch (chooseRewriteStrategy(c, session)) { - case NONE: - c = traverse(c, session); - break; - case CHEAP_AND: - c = cheapTransform(c, session); - break; - case EXPENSIVE_AND: - c = expensiveTransform((AndItem) c, session); - break; - case FLAT: - c = flatTransform(c, session); - break; - default: - break; + case NONE: + c = traverse(c, session); + break; + case CHEAP_AND: + c = cheapTransform(c, session); + break; + case EXPENSIVE_AND: + c = expensiveTransform((AndItem) c, session); + break; + case FLAT: + c = flatTransform(c, session); + break; + default: + break; } } return c; @@ -200,8 +202,7 @@ public class IndexCombinatorSearcher extends Searcher { int length = c.getItemCount(); for (int i = 0; i < length; ++i) { Item word = c.getItem(i); - if (word instanceof CompositeItem && !(word instanceof PhraseItem) - && !(word instanceof BlockItem)) { + if (word instanceof CompositeItem && !(word instanceof PhraseItem) && !(word instanceof BlockItem)) { c.setItem(i, rewrite((CompositeItem) word, session)); } } @@ -335,7 +336,7 @@ public class IndexCombinatorSearcher extends Searcher { WordItem newWord = new WordItem(asPhrase.getIndexedString(), newIndex.name, false); return newWord; } else if (word instanceof IndexedItem) { - word.setIndexName(newIndex.name); + word.setIndexName(newIndex.name); } else if (word instanceof CompositeItem) { CompositeItem asComposite = (CompositeItem) word; for (Iterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java index 72c38448936..7456f33d00f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NoRankingSearcher.java @@ -1,16 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform; - import java.util.List; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.query.Sorting.FieldOrder; import com.yahoo.search.searchchain.Execution; - /** * Avoid doing relevance calculations if sorting only * on attributes. @@ -25,7 +25,7 @@ public class NoRankingSearcher extends Searcher { private static final String UNRANKED = "unranked"; @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { List<FieldOrder> s = (query.getRanking().getSorting() != null) ? query.getRanking().getSorting().fieldOrders() : null; if (s == null) { return execution.search(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java index 7a548acbff7..ffb1b8a4965 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java @@ -5,6 +5,8 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.searchchain.Execution; @@ -12,7 +14,7 @@ import com.yahoo.search.searchchain.Execution; import java.util.List; /** - * <p>Detects and removes certain phrases from the query.</p> + * Detects and removes certain phrases from the query. * * @author bratseth */ @@ -52,9 +54,9 @@ public class NonPhrasingSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { - List<PhraseMatcher.Phrase> phrases=phraseMatcher.matchPhrases(query.getModel().getQueryTree().getRoot()); - if (phrases!=null && !query.properties().getBoolean(suggestonly, false)) { + public Result search(Query query, Execution execution) { + List<PhraseMatcher.Phrase> phrases = phraseMatcher.matchPhrases(query.getModel().getQueryTree().getRoot()); + if (phrases != null && !query.properties().getBoolean(suggestonly, false)) { remove(phrases); query.trace("Removing stop words",true,2); } @@ -64,9 +66,9 @@ public class NonPhrasingSearcher extends Searcher { private void remove(List<PhraseMatcher.Phrase> phrases) { // Removing the leaf replace phrases first to preserve // the start index of each replace phrase until removing - for (int i=phrases.size()-1; i>=0; i-- ) { - PhraseMatcher.Phrase phrase= phrases.get(i); - if (phrase.getLength()<phrase.getOwner().getItemCount()) // Don't removeField all + for (int i = phrases.size()-1; i >= 0; i-- ) { + PhraseMatcher.Phrase phrase = phrases.get(i); + if (phrase.getLength() < phrase.getOwner().getItemCount()) // Don't removeField all phrase.remove(); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index 02c8ecda60c..fdd6ad47a98 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -11,6 +11,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexFacts.Session; import com.yahoo.prelude.query.*; import com.yahoo.prelude.query.WordAlternativesItem.Alternative; +import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; @@ -46,7 +47,7 @@ public class NormalizingSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { normalize(query, execution.context().getIndexFacts().newSession(query)); return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java index f4891489216..e8e4dc39fd5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java @@ -20,15 +20,15 @@ public class PhraseMatcher { private FSA phraseFSA = null; - private boolean matchPhraseItems=false; + private boolean matchPhraseItems = false; - private boolean matchSingleItems=false; + private boolean matchSingleItems = false; /** Whether this should ignore regular plural/singular form differences when matching */ - private boolean ignorePluralForm=false; + private boolean ignorePluralForm = false; /** False to matche the longest phrase, true to match <i>all</i> phrases */ - private boolean matchAll =false; + private boolean matchAll = false; /** For null subclass only */ private PhraseMatcher() { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java index d530ec6b45e..2f3f4dbd351 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java @@ -29,7 +29,7 @@ import java.util.List; @Provides(PhrasingSearcher.PHRASE_REPLACEMENT) public class PhrasingSearcher extends Searcher { - private static final CompoundName suggestonly=new CompoundName("suggestonly"); + private static final CompoundName suggestonly = new CompoundName("suggestonly"); public static final String PHRASE_REPLACEMENT = "PhraseReplacement"; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index 4490d3c9b1e..69331a196a2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -38,7 +38,7 @@ public class RecallSearcher extends Searcher { private static final CompoundName recallName=new CompoundName("recall"); @Override - public com.yahoo.search.Result search(Query query, Execution execution) { + public Result search(Query query, Execution execution) { String recall = query.properties().getString(recallName); if (recall == null) return execution.search(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 1fad6e0b1c3..d70e0e2ff66 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -53,7 +53,7 @@ public class BlendingSearcher extends Searcher { } @Override - public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + public Result search(Query query, Execution execution) { Result result = execution.search(query); Result blended = blendResults(result, query, query.getOffset(), query.getHits(), execution); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java index 415ebd7871c..2f9e81c1607 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java @@ -27,9 +27,12 @@ import java.util.Set; * will be returned when attribute prefetch filling is requested.</p> * * @author bratseth + * @deprecated use {@link com.yahoo.search.searchchain.testutil.DocumentSourceSearcher} */ @SuppressWarnings({"rawtypes"}) +@Deprecated // TODO: Remove on Vespa 7 public class DocumentSourceSearcher extends Searcher { + // as for the SuppressWarnings annotation above, we are inside // com.yahoo.prelude, this is old stuff, really no point firing off those // warnings here... @@ -38,7 +41,6 @@ public class DocumentSourceSearcher extends Searcher { private Map<Query, Result> completelyFilledResults = new HashMap<>(); private Map<Query, Result> attributeFilledResults = new HashMap<>(); private Map<Query, Result> unFilledResults = new HashMap<>(); - //private Result defaultUnfilledResult; /** Time (in ms) at which the index of this searcher was last modified */ long editionTimeStamp=0; @@ -101,11 +103,11 @@ public class DocumentSourceSearcher extends Searcher { } /** - * Returns a query clone which has offset and hits set to null. This is used by access to + * Returns a query clone which has source, offset and hits set to null. This is used by access to * the maps using the query as key to achieve lookup independent of offset/hits value */ - private com.yahoo.search.Query getQueryKeyClone(com.yahoo.search.Query query) { - com.yahoo.search.Query key=query.clone(); + private Query getQueryKeyClone(Query query) { + Query key = query.clone(); key.setWindow(0,0); key.getModel().setSources(""); return key; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index 71e54c810c2..694c30eba9a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -19,10 +19,9 @@ import java.util.Map; /** - * A searcher which does parametrized collapsing. Based on - * SiteCollapsingSearcher. Deprecated - use grouping. + * A searcher which does parametrized collapsing. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @SuppressWarnings("deprecation") @After(PhaseNames.RAW_QUERY) @@ -96,6 +95,7 @@ public class FieldCollapsingSearcher extends Searcher { * If collapse is active, do collapsing. * Otherwise, act as a simple pass through */ + @Override public Result search(com.yahoo.search.Query query, Execution execution) { String collapseField = query.properties().getString(collapsefield); @@ -174,17 +174,18 @@ public class FieldCollapsingSearcher extends Searcher { } if (knownCollapses.containsKey(collapseId)) { - int numHitsThisField = knownCollapses.get(collapseId).intValue(); + int numHitsThisField = knownCollapses.get(collapseId); if (numHitsThisField < collapseSize) { result.hits().add(hit); ++numHitsThisField; - knownCollapses.put(collapseId, Integer.valueOf(numHitsThisField)); + knownCollapses.put(collapseId, numHitsThisField); } } else { - knownCollapses.put(collapseId, Integer.valueOf(1)); + knownCollapses.put(collapseId, 1); result.hits().add(hit); } } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java index c18f3d49da3..2330ca2382a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java @@ -16,16 +16,17 @@ import java.util.Iterator; /** * Save the query in the incoming state to a meta hit in the result. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ - public class JSONDebugSearcher extends Searcher { + public static final String JSON_FIELD = "JSON field: "; public static final String STRUCT_FIELD = "Structured data field (as json): "; public static final String FEATURE_FIELD = "Feature data field (as json): "; private static CompoundName PROPERTYNAME = new CompoundName("dumpjson"); + @Override public Result search(com.yahoo.search.Query query, Execution execution) { Result r = execution.search(query); String propertyName = query.properties().getString(PROPERTYNAME); @@ -53,4 +54,5 @@ public class JSONDebugSearcher extends Searcher { } return r; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index ca87c0c1d46..5c56379efc0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -78,13 +78,13 @@ public class JuniperSearcher extends Searcher { @Override public void fill(Result result, String summaryClass, Execution execution) { Result workResult = result; - final int worstCase = workResult.getHitCount(); - final List<Hit> hits = new ArrayList<>(worstCase); - for (final Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { - final Hit sniffHit = i.next(); + int worstCase = workResult.getHitCount(); + List<Hit> hits = new ArrayList<>(worstCase); + for (Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { + Hit sniffHit = i.next(); if ( ! (sniffHit instanceof FastHit)) continue; - final FastHit hit = (FastHit) sniffHit; + FastHit hit = (FastHit) sniffHit; if (hit.isFilled(summaryClass)) continue; hits.add(hit); @@ -105,54 +105,46 @@ public class JuniperSearcher extends Searcher { Object searchDefinitionField = hit.getField(MAGIC_FIELD); if (searchDefinitionField == null) continue; - String searchDefinitionName = searchDefinitionField.toString(); - - // TODO: Switch to iterate over indexes in the outer loop: - //for (Index index : indexFacts.getIndexes(searchDefinitionName())) { - // if (index.getDynamicSummary() || index.getHighlightSummary()) { - // insertTags(hit.buildHitField(index.getName(), true, true), bolding, index.getDynamicSummary()); - // } - //} - for (String fieldName : hit.fields().keySet()) { - Index index = indexFacts.getIndex(fieldName, searchDefinitionName); - if (index.getDynamicSummary() || index.getHighlightSummary()) - insertTags(hit.buildHitField(fieldName, true, true), bolding, index.getDynamicSummary()); + + for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { + if (index.getDynamicSummary() || index.getHighlightSummary()) { + HitField fieldValue = hit.buildHitField(index.getName(), true, true); + if (fieldValue != null) + insertTags(fieldValue, bolding, index.getDynamicSummary()); + } } } } - private void insertTags(final HitField oldProperty, final boolean bolding, final boolean dynteaser) { + private void insertTags(HitField oldProperty, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; - for (final ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { - final FieldPart f = i.next(); - if (f instanceof SeparatorFieldPart) { + for (ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { + FieldPart f = i.next(); + if (f instanceof SeparatorFieldPart) setSeparatorString(bolding, (SeparatorFieldPart) f); - } - if (f.isFinal()) { - continue; - } + if (f.isFinal()) continue; - final String toQuote = f.getContent(); + String toQuote = f.getContent(); List<FieldPart> newFieldParts = null; int previous = 0; for (int j = 0; j < toQuote.length(); j++) { - final char key = toQuote.charAt(j); + char key = toQuote.charAt(j); switch (key) { - case RAW_HIGHLIGHT_CHAR: - newFieldParts = initFieldParts(newFieldParts); - addBolding(bolding, insideHighlight, f, toQuote, newFieldParts, previous, j); - previous = j + 1; - insideHighlight = !insideHighlight; - break; - case RAW_SEPARATOR_CHAR: - newFieldParts = initFieldParts(newFieldParts); - addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, - previous, j); - previous = j + 1; - break; - default: - // no action - break; + case RAW_HIGHLIGHT_CHAR: + newFieldParts = initFieldParts(newFieldParts); + addBolding(bolding, insideHighlight, f, toQuote, newFieldParts, previous, j); + previous = j + 1; + insideHighlight = !insideHighlight; + break; + case RAW_SEPARATOR_CHAR: + newFieldParts = initFieldParts(newFieldParts); + addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, + previous, j); + previous = j + 1; + break; + default: + // no action + break; } } if (previous > 0 && previous < toQuote.length()) { @@ -160,37 +152,30 @@ public class JuniperSearcher extends Searcher { } if (newFieldParts != null) { i.remove(); - for (final Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext();) { + for (Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext();) { i.add(j.next()); } } } } - private void setSeparatorString(final boolean bolding,final SeparatorFieldPart f) { - if (bolding) { + private void setSeparatorString(boolean bolding, SeparatorFieldPart f) { + if (bolding) f.setContent(separatorTag); - } else { + else f.setContent(ELLIPSIS); - } } - private void addSeparator(final boolean bolding, final boolean dynteaser, - final FieldPart f, final String toQuote, - final List<FieldPart> newFieldParts, final int previous, final int j) { - if (previous != j) { + private void addSeparator(boolean bolding, boolean dynteaser, FieldPart f, String toQuote, + List<FieldPart> newFieldParts, int previous, int j) { + if (previous != j) newFieldParts.add(new StringFieldPart(toQuote.substring(previous, j), f.isToken())); - } - if (dynteaser) { - final FieldPart s = (bolding ? new SeparatorFieldPart(separatorTag) : new SeparatorFieldPart(ELLIPSIS)); - newFieldParts.add(s); - } + if (dynteaser) + newFieldParts.add(bolding ? new SeparatorFieldPart(separatorTag) : new SeparatorFieldPart(ELLIPSIS)); } - private void addBolding(final boolean bolding, - final boolean insideHighlight, final FieldPart f, - final String toQuote, final List<FieldPart> newFieldParts, - final int previous, final int j) { + private void addBolding(boolean bolding, boolean insideHighlight, FieldPart f, String toQuote, + List<FieldPart> newFieldParts, int previous, int j) { if (previous != j) { newFieldParts.add(new StringFieldPart(toQuote.substring(previous, j), f.isToken())); } @@ -209,9 +194,8 @@ public class JuniperSearcher extends Searcher { } private List<FieldPart> initFieldParts(List<FieldPart> newFieldParts) { - if (newFieldParts == null) { + if (newFieldParts == null) newFieldParts = new ArrayList<>(); - } return newFieldParts; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java index c47af9e32da..3b2fd596cfa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java @@ -17,7 +17,7 @@ import java.util.*; * * <p> For each group, the desired number of hits can be specified. </p> * - * @author tonytv + * @author tonytv */ public class MultipleResultsSearcher extends Searcher { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 33667349397..43717ecf6cd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -34,6 +34,7 @@ import com.yahoo.prelude.Location; @Before(PhaseNames.TRANSFORMED_QUERY) @Provides(PosSearcher.POSITION_PARSING) public class PosSearcher extends Searcher { + public static final String POSITION_PARSING = "PositionParsing"; private static final CompoundName posBb = new CompoundName("pos.bb"); @@ -52,7 +53,7 @@ public class PosSearcher extends Searcher { public final static double km2deg = 1000.000 * 180.0 / (Math.PI * 6356752.0); public final static double mi2deg = 1609.344 * 180.0 / (Math.PI * 6356752.0); - + @Override public Result search(Query query, Execution execution) { String bb = query.properties().getString(posBb); String ll = query.properties().getString(posLl); @@ -92,9 +93,8 @@ public class PosSearcher extends Searcher { } } catch (IllegalArgumentException e) { - // System.err.println("error: "+e); - return new Result(query, ErrorMessage.createInvalidQueryParameter( - "Error in pos parameters: " + Exceptions.toMessageString(e))); + return new Result(query, ErrorMessage.createInvalidQueryParameter("Error in pos parameters: " + + Exceptions.toMessageString(e))); } // and finally: query.getRanking().setLocation(loc); @@ -102,8 +102,8 @@ public class PosSearcher extends Searcher { } private void handleGeoCircle(Query query, String ll, Location target) { - double ewCoord = 0; - double nsCoord = 0; + double ewCoord; + double nsCoord; try { DegreesParser parsed = new DegreesParser(ll); ewCoord = parsed.longitude; @@ -111,9 +111,9 @@ public class PosSearcher extends Searcher { } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Unable to parse lat/long string '" +ll + "'", e); } - String radius = query.properties().getString(posRadius); - double radiusdegrees = 0.0; + String radius = query.properties().getString(posRadius); + double radiusdegrees; if (radius == null) { radiusdegrees = 50.0 * km2deg; } else if (radius.endsWith("km")) { @@ -133,8 +133,8 @@ public class PosSearcher extends Searcher { private void handleXyCircle(Query query, String xy, Location target) { - int xcoord = 0; - int ycoord = 0; + int xcoord; + int ycoord; // parse xy int semipos = xy.indexOf(';'); if (semipos > 0 && semipos < xy.length()) { @@ -143,8 +143,9 @@ public class PosSearcher extends Searcher { } else { throw new IllegalArgumentException("pos.xy must be in the format 'digits;digits' but was: '"+xy+"'"); } + String radius = query.properties().getString(posRadius); - int radiusUnits = 0; + int radiusUnits; if (radius == null) { radiusUnits = 5000; } else if (radius.endsWith("km")) { @@ -165,7 +166,6 @@ public class PosSearcher extends Searcher { target.setXyCircle(xcoord, ycoord, radiusUnits); } - private static void parseBoundingBox(String bb, Location target) { BoundingBoxParser parser = new BoundingBoxParser(bb); target.setBoundingBox(parser.n, parser.s, parser.e, parser.w); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java index 81b948682df..32efcde6feb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java @@ -11,19 +11,20 @@ import com.yahoo.search.searchchain.Execution; /** * Save the query in the incoming state to a meta hit in the result. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ - +@Deprecated // TODO: Remove on Vespa 7 public class QuerySnapshotSearcher extends Searcher { public Result search(Query query, Execution execution) { Query q = query.clone(); Result r = execution.search(query); - Hit h = new Hit("meta:querysnapshot", new Relevance( - Double.POSITIVE_INFINITY)); + Hit h = new Hit("meta:querysnapshot", new Relevance(Double.POSITIVE_INFINITY)); h.setMeta(true); h.setField("query", q); r.hits().add(h); return r; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java index 4e604dcd226..558521a7a8d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java @@ -10,8 +10,10 @@ import com.yahoo.search.searchchain.Execution; /** * Ensures hits is 1000 or less and offset is 1000 or less. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class QueryValidatingSearcher extends Searcher { public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java index d4cad7f1246..5dcc533fb1f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java @@ -35,6 +35,7 @@ public class QuotingSearcher extends Searcher { } private static class QuoteTable { + private final int lowerUncachedBound; private final int upperUncachedBound; private final Map<Character, String> quoteMap; @@ -50,12 +51,10 @@ public class QuotingSearcher extends Searcher { boolean newIsEmpty = true; Map<Character, String> newQuoteMap = new HashMap<>(); for (Iterator<?> i = config.character().iterator(); i.hasNext(); ) { - QrQuotetableConfig.Character character - = (QrQuotetableConfig.Character)i.next(); + QrQuotetableConfig.Character character = (QrQuotetableConfig.Character)i.next(); if (character.ordinal() > 256) { newIsEmpty = false; - newQuoteMap.put(new Character((char)character.ordinal()), - character.quoting()); + newQuoteMap.put(new Character((char)character.ordinal()), character.quoting()); newUseMap = true; if (minOrd == 0 || character.ordinal() < minOrd) minOrd = character.ordinal(); @@ -64,8 +63,7 @@ public class QuotingSearcher extends Searcher { } else { newIsEmpty = false; - newLowerTable[character.ordinal()] - = character.quoting(); + newLowerTable[character.ordinal()] = character.quoting(); } } lowerUncachedBound = minOrd; @@ -75,22 +73,19 @@ public class QuotingSearcher extends Searcher { isEmpty = newIsEmpty; lowerTable = newLowerTable; } + public String get(char c) { - if (isEmpty) - return null; + if (isEmpty) return null; + int ord = (int)c; if (ord < 256) { return lowerTable[ord]; } else { - if ((!useMap) || ord < lowerUncachedBound - || ord > upperUncachedBound) - { + if ((!useMap) || ord < lowerUncachedBound || ord > upperUncachedBound) return null; - } - else { + else return quoteMap.get(new Character(c)); - } } } public boolean isEmpty() { @@ -107,35 +102,29 @@ public class QuotingSearcher extends Searcher { Result result = execution.search(query); execution.fill(result); QuoteTable translations = getQuoteTable(); - if (translations == null || translations.isEmpty()) { - return result; - } + if (translations == null || translations.isEmpty()) return result; + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext(); ) { Hit h = i.next(); - if (h instanceof FastHit) { - quoteProperties((FastHit)h, translations); - } + if (h instanceof FastHit) + quoteFields((FastHit) h, translations); } return result; } - private void quoteProperties(FastHit hit, QuoteTable translations) { - for (Iterator<?> i = ((Set<?>) hit.fields().keySet()).iterator(); i.hasNext(); ) { - String propertyName = (String) i.next(); - Object entry = hit.getField(propertyName); - if (entry == null) { - continue; - } - Class<? extends Object> propertyType = entry.getClass(); - if (propertyType.equals(HitField.class)) { - quoteField((HitField) entry, translations); - } else if (propertyType.equals(String.class)) { - quoteProperty(hit, propertyName, (String)entry, translations); + private void quoteFields(FastHit hit, QuoteTable translations) { + hit.forEachField((fieldName, fieldValue) -> { + if (fieldValue != null) { + Class<?> fieldType = fieldValue.getClass(); + if (fieldType.equals(HitField.class)) + quoteField((HitField) fieldValue, translations); + else if (fieldType.equals(String.class)) + quoteField(hit, fieldName, (String) fieldValue, translations); } - } + }); } - private void quoteProperty(Hit hit, String fieldname, String toQuote, QuoteTable translations) { + private void quoteField(Hit hit, String fieldname, String toQuote, QuoteTable translations) { List<FieldPart> l = translate(toQuote, translations, true); if (l != null) { HitField hf = new HitField(fieldname, toQuote); @@ -144,13 +133,11 @@ public class QuotingSearcher extends Searcher { } } - private void quoteField(HitField field, QuoteTable translations) { for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext(); ) { FieldPart f = i.next(); - if (!f.isFinal()) { - List<FieldPart> newFieldParts = translate(f.getContent(), translations, - f.isToken()); + if ( ! f.isFinal()) { + List<FieldPart> newFieldParts = translate(f.getContent(), translations, f.isToken()); if (newFieldParts != null) { i.remove(); for (Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext(); ) { @@ -161,33 +148,24 @@ public class QuotingSearcher extends Searcher { } } - private List<FieldPart> translate(String toQuote, QuoteTable translations, - boolean isToken) { + private List<FieldPart> translate(String toQuote, QuoteTable translations, boolean isToken) { List<FieldPart> newFieldParts = null; int lastIdx = 0; for (int i = 0; i < toQuote.length(); i++) { String quote = translations.get(toQuote.charAt(i)); if (quote != null) { - if (newFieldParts == null) { + if (newFieldParts == null) newFieldParts = new ArrayList<>(); - } - if (lastIdx != i) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx, i), - isToken)); - } + if (lastIdx != i) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx, i), isToken)); String initContent = Character.toString(toQuote.charAt(i)); - newFieldParts.add(new ImmutableFieldPart(initContent, - quote, - isToken)); + newFieldParts.add(new ImmutableFieldPart(initContent, quote, isToken)); lastIdx = i+1; } } - if (lastIdx > 0 && lastIdx < toQuote.length()) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx), - isToken)); - } + if (lastIdx > 0 && lastIdx < toQuote.length()) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx), isToken)); return newFieldParts; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java index 2e2c73b6707..9b6f5926b61 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java @@ -20,7 +20,7 @@ import java.util.Collection; /** * Checks that predicate queries don't use values outside the defined upper/lower bounds. * - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + * @author Magnar Nedland */ @After(BooleanSearcher.PREDICATE) public class ValidatePredicateSearcher extends Searcher { @@ -78,4 +78,5 @@ public class ValidatePredicateSearcher extends Searcher { @Override public void onExit() {} } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 9de1a5e2a2d..bd61de7f783 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -10,6 +10,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.Coverage; @@ -207,7 +208,8 @@ public class StatisticsSearcher extends Searcher { * 2) Add response time to total response time (time from entry to return) * 3) ..... */ - public Result search(com.yahoo.search.Query query, Execution execution) { + @Override + public Result search(Query query, Execution execution) { if (query.properties().getBoolean(IGNORE_QUERY,false)) { return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java index 7a904ea014c..7989f35e77b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java @@ -9,7 +9,10 @@ import com.yahoo.text.XML; * A set of variable bindings for template rendering * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public abstract class Context { private boolean xmlEscape = true; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java index f14d8ddf319..dab80580f61 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java @@ -13,7 +13,10 @@ import java.util.Set; * Defines formatting options used with special kinds of hits. * * @author laboisse + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class FormattingOptions { public static final String DEFAULT_TYPE_ATTRIBUTE_NAME = "type"; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java index 037d1a77d5c..4d1daa97306 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java @@ -17,7 +17,10 @@ import java.util.Set; * A context providing all the fields of a hit, and falls back to MapContext behavior for all other keys. * * @author tonytv + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class HitContext extends Context { private final Hit hit; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java index 49c5ffa6e78..84d97b71f60 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java @@ -6,7 +6,12 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -/** A context having a map as secondary storage */ +/** + * A context having a map as secondary storage + * @deprecated use a Renderer instead + */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class MapContext extends Context { private Map<String, Object> map = new LinkedHashMap<>(); diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java index a24fd623e4d..83118ec66ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java @@ -14,7 +14,10 @@ import java.io.Writer; * This is a variant of the tiled template set - see that class for details. * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class PageTemplateSet extends TiledTemplateSet { public PageTemplateSet() { diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java index 31e133d22d5..a639a6b97ec 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java @@ -21,8 +21,10 @@ import java.util.Iterator; * Renders a search result using the old templates API. * * @author tonytv + * @deprecated do not use */ @SuppressWarnings({ "rawtypes", "deprecation", "unchecked" }) +@Deprecated // TODO: Remove on Vespa 7 public final class SearchRendererAdaptor extends Renderer { private final LogExceptionUserTemplateDelegator templates; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java index 63bd3214b17..3d00be9d05b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java @@ -10,7 +10,10 @@ import java.io.Writer; * template mechanism by subclassing this. * * @author bratseth + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public abstract class Template<T extends Writer> { /** @@ -19,8 +22,7 @@ public abstract class Template<T extends Writer> { * @param context the context to evaluate in * @param writer the writer to render to */ - public abstract void render(Context context,T writer) - throws java.io.IOException; + public abstract void render(Context context,T writer) throws java.io.IOException; /** diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java index 6bba0f620ee..91bc33e3e2a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java @@ -70,7 +70,10 @@ import java.util.stream.Collectors; * * @author bratseth * @author laboisse + * @deprecated use a Renderer instead */ +@SuppressWarnings("deprecation") +@Deprecated // TODO: Remove on Vespa 7 public class TiledTemplateSet extends DefaultTemplateSet { private FormattingOptions hitOptionsForProvider; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index f13fb2e88f4..ab6976e29d9 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -351,10 +351,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } /** - * Creates a new query from another query, but with time sensitive - * fields reset. - * - * @return new query + * Creates a new query from another query, but with time sensitive fields reset. */ public static Query createNewQuery(Query query) { return new Query(query, System.currentTimeMillis()); diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 686c019688e..b32eec876cc 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -261,6 +261,7 @@ public class FederationSearcher extends ForkingSearcher { } private Query cloneFederationQuery(Query query, Window window, long timeout, Target target) { + query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once Query clonedQuery = Query.createNewQuery(query); return createFederationQuery(query, clonedQuery, window, timeout, target); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java index 727b24a39f7..dfa4f9ad9bb 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java @@ -158,6 +158,7 @@ public class VespaSearcher extends ConfiguredHTTPProviderSearcher { return marshalQuery(query.getModel().getQueryTree()); } + query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once Query workQuery = query.clone(); String error = QueryCanonicalizer.canonicalize(workQuery); if (error != null) { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 1c6e92ebffc..e9e4e34727c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -230,7 +230,7 @@ public class SearchHandler extends LoggingRequestHandler { return (e.getCause() instanceof IllegalArgumentException) ? invalidParameterResponse(request, e) : illegalQueryResponse(request, e); - } catch (RuntimeException e) { // Make sure we generate a valid XML response even on unexpected errors + } catch (RuntimeException e) { // Make sure we generate a valid response even on unexpected errors log.log(Level.WARNING, "Failed handling " + request, e); return internalServerErrorResponse(request, e); } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java index e10c67e5ff5..2d0ff0c62db 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java @@ -295,6 +295,7 @@ public abstract class QueryRewriteSearcher extends Searcher { // Store rewriter result HashMap<String, Object> rewriterResult = null; + query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once Query originalQueryObj = query.clone(); try { diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 2768a546cd0..399ff6194c8 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -40,8 +40,8 @@ public class NGramSearcher extends Searcher { private final CharacterClasses characterClasses; public NGramSearcher(Linguistics linguistics) { - gramSplitter= linguistics.getGramSplitter(); - characterClasses= linguistics.getCharacterClasses(); + gramSplitter = linguistics.getGramSplitter(); + characterClasses = linguistics.getCharacterClasses(); } @Override @@ -54,7 +54,7 @@ public class NGramSearcher extends Searcher { if (rewritten) query.trace("Rewritten to n-gram matching",true,2); - Result result=execution.search(query); + Result result = execution.search(query); recombineNGrams(result.hits().deepIterator(), session); return result; } @@ -160,10 +160,11 @@ public class NGramSearcher extends Searcher { if (hit.isMeta()) continue; Object sddocname = hit.getField(Hit.SDDOCNAME_FIELD); if (sddocname == null) return; - for (String fieldName : hit.fieldKeys()) { // TODO: Iterate over indexes instead - Index index = session.getIndex(fieldName, sddocname.toString()); + for (Index index : session.getIndexes(sddocname.toString())) { if (index.isNGram() && (index.getHighlightSummary() || index.getDynamicSummary())) { - hit.setField(fieldName, recombineNGramsField(hit.getField(fieldName), index.getGramSize())); + Object fieldValue = hit.getField(index.getName()); + if (fieldValue != null) + hit.setField(index.getName(), recombineNGramsField(fieldValue, index.getGramSize())); } } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 6c7018317c3..55c846ccb5b 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -68,6 +68,7 @@ import java.util.function.LongSupplier; * JSON renderer for search results. * * @author Steinar Knutsen + * @author bratseth */ // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @@ -75,18 +76,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); - private enum RenderDecision { - YES, NO, DO_NOT_KNOW; - - boolean booleanValue() { - switch (this) { - case YES: return true; - case NO: return false; - default: throw new IllegalStateException(); - } - } - } - // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; private static final String BUCKET_TO = "to"; @@ -133,6 +122,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final JsonFactory generatorFactory; private JsonGenerator generator; + private FieldConsumer fieldConsumer; private Deque<Integer> renderedChildren; private boolean debugRendering; private LongSupplier timeSource; @@ -304,9 +294,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void init() { super.init(); - generator = null; - renderedChildren = null; debugRendering = false; + setGenerator(null, debugRendering); + renderedChildren = null; timeSource = System::currentTimeMillis; stream = null; } @@ -314,9 +304,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); - generator = generatorFactory.createGenerator(stream, JsonEncoding.UTF8); - renderedChildren = new ArrayDeque<>(); debugRendering = getDebugRendering(getResult().getQuery()); + setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); + renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); renderTrace(getExecution().trace()); renderTiming(); @@ -473,17 +463,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return ! (hit instanceof DefaultErrorHit); } - private void fieldsStart(MutableBoolean hasFieldsField) throws IOException { - if (hasFieldsField.get()) return; - generator.writeObjectFieldStart(FIELDS); - hasFieldsField.set(true); - } - - private void fieldsEnd(MutableBoolean hasFieldsField) throws IOException { - if ( ! hasFieldsField.get()) return; - generator.writeEndObject(); - } - private void renderHitContents(Hit hit) throws IOException { String id = hit.getDisplayId(); if (id != null) @@ -509,39 +488,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } private void renderAllFields(Hit hit) throws IOException { - MutableBoolean hasFieldsField = new MutableBoolean(false); - renderTotalHitCount(hit, hasFieldsField); - renderStandardFields(hit, hasFieldsField); - fieldsEnd(hasFieldsField); - } - - private void renderStandardFields(Hit hit, MutableBoolean hasFieldsField) { - hit.forEachField((name, value) -> { - try { - if (shouldRender(name, value)) { - fieldsStart(hasFieldsField); - renderField(name, value); - } - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + fieldConsumer.startHitFields(); + renderTotalHitCount(hit); + renderStandardFields(hit); + fieldConsumer.endHitFields(); } - private boolean shouldRender(String name, Object value) { - if (debugRendering) return true; - - if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; - - if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; - - // StringFieldValue cannot hold a null, so checking length directly is OK: - if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; - - if (value instanceof NanNumber) return false; - - return true; + private void renderStandardFields(Hit hit) { + hit.forEachFieldAsRaw(fieldConsumer); } private void renderSpecialCasesForGrouping(Hit hit) throws IOException { @@ -606,96 +560,13 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getTo()) : id.getTo()).toString(); } - private void renderTotalHitCount(Hit hit, MutableBoolean hasFieldsField) throws IOException { + private void renderTotalHitCount(Hit hit) throws IOException { if ( ! (getRecursionLevel() == 1 && hit instanceof HitGroup)) return; - fieldsStart(hasFieldsField); + fieldConsumer.ensureFieldsField(); generator.writeNumberField(TOTAL_COUNT, getResult().getTotalHitCount()); - } - - private void renderField(String name, Object value) throws IOException { - generator.writeFieldName(name); - renderFieldContents(value); - } - - private void renderFieldContents(Object field) throws IOException { - if (field == null) { - generator.writeNull(); - } else if (field instanceof Number) { - renderNumberField((Number) field); - } else if (field instanceof TreeNode) { - generator.writeTree((TreeNode) field); - } else if (field instanceof Tensor) { - renderTensor(Optional.of((Tensor)field)); - } else if (field instanceof JsonProducer) { - generator.writeRawValue(((JsonProducer) field).toJson()); - } else if (field instanceof Inspectable) { - StringBuilder intermediate = new StringBuilder(); - JsonRender.render((Inspectable) field, intermediate, true); - generator.writeRawValue(intermediate.toString()); - } else if (field instanceof StringFieldValue) { - // This needs special casing as JsonWriter hides empty strings now - generator.writeString(((StringFieldValue)field).getString()); - } else if (field instanceof TensorFieldValue) { - renderTensor(((TensorFieldValue)field).getTensor()); - } else if (field instanceof FieldValue) { - // the null below is the field which has already been written - ((FieldValue) field).serialize(null, new JsonWriter(generator)); - } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in - // syntactically correct JSON - String s = field.toString(); - if (s == null) { - generator.writeNull(); - } else { - generator.writeRawValue(s); - } - } else { - generator.writeString(field.toString()); - } - } - - private void renderNumberField(Number field) throws IOException { - if (field instanceof Integer) { - generator.writeNumber(field.intValue()); - } else if (field instanceof Float) { - generator.writeNumber(field.floatValue()); - } else if (field instanceof Double) { - generator.writeNumber(field.doubleValue()); - } else if (field instanceof Long) { - generator.writeNumber(field.longValue()); - } else if (field instanceof Byte || field instanceof Short) { - generator.writeNumber(field.intValue()); - } else if (field instanceof BigInteger) { - generator.writeNumber((BigInteger) field); - } else if (field instanceof BigDecimal) { - generator.writeNumber((BigDecimal) field); - } else { - generator.writeNumber(field.doubleValue()); - } - } - - private void renderTensor(Optional<Tensor> tensor) throws IOException { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - if (tensor.isPresent()) { - for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { - Tensor.Cell cell = i.next(); - - generator.writeStartObject(); - - generator.writeObjectFieldStart("address"); - for (int d = 0; d < cell.getKey().size(); d++) - generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); - generator.writeEndObject(); - - generator.writeObjectField("value", cell.getValue()); - - generator.writeEndObject(); - } - } - generator.writeEndArray(); - generator.writeEndObject(); + // alternative for the above two lines: + // fieldConsumer.accept(TOTAL_COUNT, getResult().getTotalHitCount()); } @Override @@ -774,11 +645,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return null; } - /** - * Only for testing. Never to be used in any other context. - */ - void setGenerator(JsonGenerator generator) { + private void setGenerator(JsonGenerator generator, boolean debugRendering) { this.generator = generator; + this.fieldConsumer = generator == null ? null : new FieldConsumer(generator, debugRendering); } /** @@ -787,5 +656,170 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { void setTimeSource(LongSupplier timeSource) { this.timeSource = timeSource; } - + + /** + * Received callbacks when fields of hits are encountered. + * This instance is reused for all hits of a Result since we are in a single-threaded context + * and want to limit object creation. + */ + private static class FieldConsumer implements Hit.RawUtf8Consumer { + + private final JsonGenerator generator; + private final boolean debugRendering; + + private MutableBoolean hasFieldsField; + + public FieldConsumer(JsonGenerator generator, boolean debugRendering) { + this.generator = generator; + this.debugRendering = debugRendering; + } + + /** + * Call before using this for a hit to track whether we + * have created the "fields" field of the JSON object + */ + void startHitFields() { + this.hasFieldsField = new MutableBoolean(false); + } + + /** Call before rendering a field to the generator */ + void ensureFieldsField() throws IOException { + if (hasFieldsField.get()) return; + generator.writeObjectFieldStart(FIELDS); + hasFieldsField.set(true); + } + + /** Call after all fields in a hit to close the "fields" field of the JSON object */ + void endHitFields() throws IOException { + if ( ! hasFieldsField.get()) return; + generator.writeEndObject(); + this.hasFieldsField = null; + } + + @Override + public void accept(String name, Object value) { + try { + if (shouldRender(name, value)) { + ensureFieldsField(); + generator.writeFieldName(name); + renderFieldContents(value); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void accept(String name, byte[] utf8Data, int offset, int length) { + try { + if (shouldRenderUtf8Value(name, length)) { + ensureFieldsField(); + generator.writeFieldName(name); + generator.writeUTF8String(utf8Data, offset, length); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private boolean shouldRender(String name, Object value) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; + // StringFieldValue cannot hold a null, so checking length directly is OK: + if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; + if (value instanceof NanNumber) return false; + return true; + } + + private boolean shouldRenderUtf8Value(String name, int length) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (length == 0) return false; + return true; + } + + private void renderFieldContents(Object field) throws IOException { + if (field == null) { + generator.writeNull(); + } else if (field instanceof Number) { + renderNumberField((Number) field); + } else if (field instanceof TreeNode) { + generator.writeTree((TreeNode) field); + } else if (field instanceof Tensor) { + renderTensor(Optional.of((Tensor)field)); + } else if (field instanceof JsonProducer) { + generator.writeRawValue(((JsonProducer) field).toJson()); + } else if (field instanceof Inspectable) { + StringBuilder intermediate = new StringBuilder(); + JsonRender.render((Inspectable) field, intermediate, true); + generator.writeRawValue(intermediate.toString()); + } else if (field instanceof StringFieldValue) { + generator.writeString(((StringFieldValue)field).getString()); + } else if (field instanceof TensorFieldValue) { + renderTensor(((TensorFieldValue)field).getTensor()); + } else if (field instanceof FieldValue) { + // the null below is the field which has already been written + ((FieldValue) field).serialize(null, new JsonWriter(generator)); + } else if (field instanceof JSONArray || field instanceof JSONObject) { + // org.json returns null if the object would not result in + // syntactically correct JSON + String s = field.toString(); + if (s == null) { + generator.writeNull(); + } else { + generator.writeRawValue(s); + } + } else { + generator.writeString(field.toString()); + } + } + + private void renderNumberField(Number field) throws IOException { + if (field instanceof Integer) { + generator.writeNumber(field.intValue()); + } else if (field instanceof Float) { + generator.writeNumber(field.floatValue()); + } else if (field instanceof Double) { + generator.writeNumber(field.doubleValue()); + } else if (field instanceof Long) { + generator.writeNumber(field.longValue()); + } else if (field instanceof Byte || field instanceof Short) { + generator.writeNumber(field.intValue()); + } else if (field instanceof BigInteger) { + generator.writeNumber((BigInteger) field); + } else if (field instanceof BigDecimal) { + generator.writeNumber((BigDecimal) field); + } else { + generator.writeNumber(field.doubleValue()); + } + } + + private void renderTensor(Optional<Tensor> tensor) throws IOException { + generator.writeStartObject(); + generator.writeArrayFieldStart("cells"); + if (tensor.isPresent()) { + for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { + Tensor.Cell cell = i.next(); + + generator.writeStartObject(); + + generator.writeObjectFieldStart("address"); + for (int d = 0; d < cell.getKey().size(); d++) + generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); + generator.writeEndObject(); + + generator.writeObjectField("value", cell.getValue()); + + generator.writeEndObject(); + } + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index f68916c8a68..74c31aa33c5 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -404,13 +404,25 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** * Receive a callback on the given object for each field in this hit. - * This is the most resource efficient way of traversing all the fields of a hit. + * This is more efficient than accessing the fields as a map or iterator. */ public void forEachField(BiConsumer<String, Object> consumer) { if (fields == null) return; fields.forEach(consumer); } + /** + * Receive a callback on the given object for each field in this hit, + * where the callback will provide raw utf-8 byte data for strings whose data + * is already available at this form. + * This is the most resource efficient way of traversing all the fields of a hit + * in renderers which produces utf-8. + */ + public void forEachFieldAsRaw(RawUtf8Consumer consumer) { + if (fields == null) return; + fields.forEach(consumer); // No utf-8 fields available in Hit + } + /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ public Map<String, Object> fields() { return getUnmodifiableFieldMap(); } @@ -800,4 +812,18 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return "hit " + getId() + " (relevance " + getRelevance() + ")"; } + public interface RawUtf8Consumer extends BiConsumer<String, Object> { + + /** + * Called for fields which are available as UTF-8 instead of accept(String, Object). + * + * @param fieldName the name of the field + * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data + * @param offset the start index of the data to accept into the utf8Data array + * @param length the length of the data to accept into the utf8Data array + */ + void accept(String fieldName, byte[] utf8Data, int offset, int length); + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java b/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java index a2159033d91..d035973a065 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java @@ -11,7 +11,7 @@ import java.util.List; * A hit orderer which can be assigned to a HitGroup to keep that group's * hit sorted in accordance with the sorting specification given when this is created. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class HitSortOrderer extends HitOrderer { diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java index 47f40f3c7f5..9e191a1219c 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Templating.java +++ b/container-search/src/main/java/com/yahoo/search/result/Templating.java @@ -15,8 +15,10 @@ import com.yahoo.search.query.Presentation; * Helper methods and data store for result attributes geared towards result * rendering and presentation. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated do not use */ +@Deprecated // TODO: Remove on Vespa 7 public class Templating { private final Result result; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index fe3d48ccded..cfef91ee0ec 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -23,8 +23,6 @@ import java.util.*; public class VespaSearchers { public static final Collection<ChainedComponentModel> vespaSearcherModels = toSearcherModels( - com.yahoo.prelude.querytransform.IndexCombinatorSearcher.class, - //com.yahoo.prelude.querytransform.LocalitySearcher.class, com.yahoo.prelude.querytransform.PhrasingSearcher.class, com.yahoo.prelude.searcher.FieldCollapsingSearcher.class, com.yahoo.search.yql.MinimalQueryInserter.class, diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index 56a6a702962..f4973ba4239 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -28,7 +28,7 @@ import com.yahoo.search.searchchain.Execution; * Any field in the configured hits which has a name starting by attribute * will be returned when attribute prefetch filling is requested.</p> * - * @author bratseth + * @author bratseth */ public class DocumentSourceSearcher extends Searcher { @@ -85,7 +85,6 @@ public class DocumentSourceSearcher extends Searcher { private void addDefaultResults() { Query q = new Query("?query=default"); Result r = new Result(q); - // These four used to assign collapseId 1,2,3,4 - re-add that if needed r.hits().add(new Hit("http://default-1.html", 0)); r.hits().add(new Hit("http://default-2.html", 0)); r.hits().add(new Hit("http://default-3.html", 0)); @@ -97,8 +96,7 @@ public class DocumentSourceSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { queryCount++; - Result r; - r = unFilledResults.get(getQueryKeyClone(query)); + Result r = unFilledResults.get(getQueryKeyClone(query)); if (r == null) { r = defaultFilledResult.clone(); } else { @@ -111,12 +109,13 @@ public class DocumentSourceSearcher extends Searcher { } /** - * Returns a query clone which has offset and hits set to null. This is used by access to + * Returns a query clone which has sourcr, offset and hits set to null. This is used by access to * the maps using the query as key to achieve lookup independent of offset/hits value */ private Query getQueryKeyClone(Query query) { - Query key=query.clone(); + Query key = query.clone(); key.setWindow(0,0); + key.getModel().setSources(""); return key; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 283a70c478b..12aec81a5f8 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -30,6 +30,7 @@ import static com.yahoo.search.yql.YqlParser.PREFIX; import static com.yahoo.search.yql.YqlParser.RANGE; import static com.yahoo.search.yql.YqlParser.RANK; import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; import static com.yahoo.search.yql.YqlParser.STEM; @@ -79,6 +80,7 @@ import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentingRule; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; @@ -106,8 +108,7 @@ public class VespaSerializer { // TODO refactor, too much copy/paste private static class AndSegmentSerializer extends Serializer { - private static void serializeWords(StringBuilder destination, - AndSegmentItem segment) { + private static void serializeWords(StringBuilder destination, AndSegmentItem segment) { for (int i = 0; i < segment.getItemCount(); ++i) { if (i > 0) { destination.append(", "); @@ -115,28 +116,23 @@ public class VespaSerializer { Item current = segment.getItem(i); if (current instanceof WordItem) { destination.append('"'); - escape(((WordItem) current).getIndexedString(), destination) - .append('"'); + escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { - throw new IllegalArgumentException( - "Serializing of " - + current.getClass().getSimpleName() + throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + " in segment AND expressions not implemented, please report this as a bug."); } } } @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { return serialize(destination, item, true); } - static boolean serialize(StringBuilder destination, Item item, - boolean includeField) { + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { AndSegmentItem phrase = (AndSegmentItem) item; Substring origin = phrase.getOrigin(); String image; @@ -154,13 +150,11 @@ public class VespaSerializer { } if (includeField) { - destination.append(normalizeIndexName(phrase.getIndexName())) - .append(" contains "); + destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); } destination.append("([{"); serializeOrigin(destination, image, offset, length); - destination.append(", \"").append(AND_SEGMENTING) - .append("\": true"); + destination.append(", \"").append(AND_SEGMENTING).append("\": true"); destination.append("}]"); destination.append(PHRASE).append('('); serializeWords(destination, phrase); @@ -189,13 +183,11 @@ public class VespaSerializer { private static class DotProductSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, DOT_PRODUCT, - (WeightedSetItem) item); + serializeWeightedSetContents(destination, DOT_PRODUCT, (WeightedSetItem) item); return false; } @@ -203,8 +195,7 @@ public class VespaSerializer { private static class EquivSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -240,8 +231,7 @@ public class VespaSerializer { private static class NearSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -304,8 +294,7 @@ public class VespaSerializer { private static class NullSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -319,31 +308,22 @@ public class VespaSerializer { private static class NumberSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { IntItem intItem = (IntItem) item; - if (intItem.getFromLimit().number() - .equals(intItem.getToLimit().number())) { - destination.append(normalizeIndexName(intItem.getIndexName())) - .append(" = "); - annotatedNumberImage(intItem, intItem.getFromLimit().number() - .toString(), destination); + if (intItem.getFromLimit().number().equals(intItem.getToLimit().number())) { + destination.append(normalizeIndexName(intItem.getIndexName())).append(" = "); + annotatedNumberImage(intItem, intItem.getFromLimit().number().toString(), destination); } else if (intItem.getFromLimit().isInfinite()) { destination.append(normalizeIndexName(intItem.getIndexName())); - destination.append(intItem.getToLimit().isInclusive() ? " <= " - : " < "); - annotatedNumberImage(intItem, intItem.getToLimit().number() - .toString(), destination); + destination.append(intItem.getToLimit().isInclusive() ? " <= " : " < "); + annotatedNumberImage(intItem, intItem.getToLimit().number().toString(), destination); } else if (intItem.getToLimit().isInfinite()) { destination.append(normalizeIndexName(intItem.getIndexName())); - destination - .append(intItem.getFromLimit().isInclusive() ? " >= " - : " > "); - annotatedNumberImage(intItem, intItem.getFromLimit().number() - .toString(), destination); + destination.append(intItem.getFromLimit().isInclusive() ? " >= " : " > "); + annotatedNumberImage(intItem, intItem.getFromLimit().number().toString(), destination); } else { serializeAsRange(destination, intItem); } @@ -358,21 +338,17 @@ public class VespaSerializer { int initLen; if (leftOpen && rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN - + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN + "\""; } else if (leftOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" - + BOUNDS_LEFT_OPEN + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_LEFT_OPEN + "\""; } else if (rightOpen) { - boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" - + BOUNDS_RIGHT_OPEN + "\""; + boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_RIGHT_OPEN + "\""; } if (annotations.length() > 0 || boundsAnnotation.length() > 0) { destination.append("[{"); } initLen = destination.length(); if (annotations.length() > 0) { - destination.append(annotations); } comma(destination, initLen); @@ -389,8 +365,7 @@ public class VespaSerializer { .append(")"); } - private void annotatedNumberImage(IntItem item, String rawNumber, - StringBuilder image) { + private void annotatedNumberImage(IntItem item, String rawNumber, StringBuilder image) { String annotations = leafAnnotations(item); if (annotations.length() > 0) { @@ -430,16 +405,14 @@ public class VespaSerializer { private static class RegExpSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { RegExpItem regexp = (RegExpItem) item; String annotations = leafAnnotations(regexp); - destination.append(normalizeIndexName(regexp.getIndexName())).append( - " matches "); + destination.append(normalizeIndexName(regexp.getIndexName())).append(" matches "); annotatedTerm(destination, regexp, annotations); return false; } @@ -498,8 +471,7 @@ public class VespaSerializer { private static class PhraseSegmentSerializer extends Serializer { - private static void serializeWords(StringBuilder destination, - PhraseSegmentItem segment) { + private static void serializeWords(StringBuilder destination, PhraseSegmentItem segment) { for (int i = 0; i < segment.getItemCount(); ++i) { if (i > 0) { destination.append(", "); @@ -507,20 +479,16 @@ public class VespaSerializer { Item current = segment.getItem(i); if (current instanceof WordItem) { destination.append('"'); - escape(((WordItem) current).getIndexedString(), destination) - .append('"'); + escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { - throw new IllegalArgumentException( - "Serializing of " - + current.getClass().getSimpleName() - + " in phrases not implemented, please report this as a bug."); + throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + + " in phrases not implemented, please report this as a bug."); } } } @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -535,8 +503,7 @@ public class VespaSerializer { int length; if (includeField) { - destination.append(normalizeIndexName(phrase.getIndexName())) - .append(" contains "); + destination.append(normalizeIndexName(phrase.getIndexName())).append(" contains "); } if (origin == null) { image = phrase.getRawWord(); @@ -555,8 +522,7 @@ public class VespaSerializer { destination.append(", ").append(annotations); } if (phrase.getSegmentingRule() == SegmentingRule.BOOLEAN_AND) { - destination.append(", ").append('"').append(AND_SEGMENTING) - .append("\": true"); + destination.append(", ").append('"').append(AND_SEGMENTING).append("\": true"); } destination.append("}]"); destination.append(PHRASE).append('('); @@ -568,16 +534,14 @@ public class VespaSerializer { private static class PhraseSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { return serialize(destination, item, true); } - static boolean serialize(StringBuilder destination, Item item, - boolean includeField) { + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { PhraseItem phrase = (PhraseItem) item; String annotations = leafAnnotations(phrase); @@ -598,11 +562,9 @@ public class VespaSerializer { } Item current = phrase.getItem(i); if (current instanceof WordItem) { - WordSerializer.serializeWordWithoutIndex(destination, - current); + WordSerializer.serializeWordWithoutIndex(destination, current); } else if (current instanceof PhraseSegmentItem) { - PhraseSegmentSerializer.serialize(destination, current, - false); + PhraseSegmentSerializer.serialize(destination, current, false); } else if (current instanceof WordAlternativesItem) { WordAlternativesSerializer.serialize(destination, (WordAlternativesItem) current, false); } else { @@ -621,16 +583,54 @@ public class VespaSerializer { } - private static class PredicateQuerySerializer extends Serializer { + private static class SameElementSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { + void onExit(StringBuilder destination, Item item) { } + + @Override + boolean serialize(StringBuilder destination, Item item) { + return serialize(destination, item, true); } + static boolean serialize(StringBuilder destination, Item item, boolean includeField) { + + SameElementItem sameElement = (SameElementItem) item; + + if (includeField) { + destination.append(normalizeIndexName(sameElement.getFieldName())).append(" contains "); + } + + destination.append(SAME_ELEMENT).append('('); + for (int i = 0; i < sameElement.getItemCount(); ++i) { + if (i > 0) { + destination.append(", "); + } + Item current = sameElement.getItem(i); + if (current instanceof WordItem) { + WordItem modified = (WordItem)current.clone(); + modified.setIndexName(sameElement.extractSubFieldName(modified)); + new WordSerializer().serialize(destination, modified); + } else { + throw new IllegalArgumentException( + "Serializing of " + current.getClass().getSimpleName() + + " in same_element is not implemented, please report this as a bug."); + } + } + destination.append(')'); + + return false; + } + + } + + private static class PredicateQuerySerializer extends Serializer { + @Override + void onExit(StringBuilder destination, Item item) { } + @Override boolean serialize(StringBuilder destination, Item item) { PredicateQueryItem pItem = (PredicateQueryItem) item; - destination.append("predicate(").append(pItem.getIndexName()) - .append(','); + destination.append("predicate(").append(pItem.getIndexName()).append(','); appendFeatures(destination, pItem.getFeatures()); destination.append(','); appendFeatures(destination, pItem.getRangeFeatures()); @@ -638,8 +638,7 @@ public class VespaSerializer { return false; } - private void appendFeatures(StringBuilder destination, - Collection<? extends PredicateQueryItem.EntryBase> features) { + private void appendFeatures(StringBuilder destination, Collection<? extends PredicateQueryItem.EntryBase> features) { if (features.isEmpty()) { destination.append('0'); // Workaround for empty maps. return; @@ -651,8 +650,7 @@ public class VespaSerializer { destination.append(','); } if (entry.getSubQueryBitmap() != PredicateQueryItem.ALL_SUB_QUERIES) { - destination.append("\"0x").append( - Long.toHexString(entry.getSubQueryBitmap())); + destination.append("\"0x").append(Long.toHexString(entry.getSubQueryBitmap())); destination.append("\":{"); appendKeyValue(destination, entry); destination.append('}'); @@ -664,19 +662,16 @@ public class VespaSerializer { destination.append('}'); } - private void appendKeyValue(StringBuilder destination, - PredicateQueryItem.EntryBase entry) { + private void appendKeyValue(StringBuilder destination, PredicateQueryItem.EntryBase entry) { destination.append('"'); escape(entry.getKey(), destination); destination.append("\":"); if (entry instanceof PredicateQueryItem.Entry) { destination.append('"'); - escape(((PredicateQueryItem.Entry) entry).getValue(), - destination); + escape(((PredicateQueryItem.Entry) entry).getValue(), destination); destination.append('"'); } else { - destination.append(((PredicateQueryItem.RangeEntry) entry) - .getValue()); + destination.append(((PredicateQueryItem.RangeEntry) entry).getValue()); destination.append('L'); } } @@ -685,8 +680,7 @@ public class VespaSerializer { private static class RangeSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -737,8 +731,7 @@ public class VespaSerializer { private static class WordAlternativesSerializer extends Serializer { @Override - void onExit(StringBuilder destination, Item item) { - } + void onExit(StringBuilder destination, Item item) { } @Override boolean serialize(StringBuilder destination, Item item) { @@ -800,10 +793,8 @@ public class VespaSerializer { abstract void onExit(StringBuilder destination, Item item); String separator(Deque<SerializerWrapper> state) { - throw new UnsupportedOperationException( - "Having several items for this query operator serializer, " - + this.getClass().getSimpleName() - + ", not yet implemented."); + throw new UnsupportedOperationException("Having several items for this query operator serializer, " + + this.getClass().getSimpleName() + ", not yet implemented."); } abstract boolean serialize(StringBuilder destination, Item item); @@ -822,8 +813,7 @@ public class VespaSerializer { } - private static final class TokenComparator implements - Comparator<Entry<Object, Integer>> { + private static final class TokenComparator implements Comparator<Entry<Object, Integer>> { @SuppressWarnings({ "rawtypes", "unchecked" }) @Override @@ -858,8 +848,7 @@ public class VespaSerializer { Serializer doIt = dispatch.get(item.getClass()); if (doIt == null) { - throw new IllegalArgumentException(item.getClass() - + " not supported for YQL+ marshalling."); + throw new IllegalArgumentException(item.getClass() + " not supported for YQL+ marshalling."); } if (state.peekFirst() != null && state.peekFirst().subItems > 0) { @@ -878,9 +867,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, WAND, - (WeightedSetItem) item, - specificAnnotations((WandItem) item)); + serializeWeightedSetContents(destination, WAND, (WeightedSetItem) item, specificAnnotations((WandItem) item)); return false; } @@ -890,18 +877,15 @@ public class VespaSerializer { double scoreThreshold = w.getScoreThreshold(); double thresholdBoostFactor = w.getThresholdBoostFactor(); if (targetNumHits != 10) { - annotations.append('"').append(TARGET_NUM_HITS).append("\": ") - .append(targetNumHits); + annotations.append('"').append(TARGET_NUM_HITS).append("\": ").append(targetNumHits); } if (scoreThreshold != 0) { comma(annotations, 0); - annotations.append('"').append(SCORE_THRESHOLD).append("\": ") - .append(scoreThreshold); + annotations.append('"').append(SCORE_THRESHOLD).append("\": ").append(scoreThreshold); } if (thresholdBoostFactor != 1) { comma(annotations, 0); - annotations.append('"').append(THRESHOLD_BOOST_FACTOR) - .append("\": ").append(thresholdBoostFactor); + annotations.append('"').append(THRESHOLD_BOOST_FACTOR).append("\": ").append(thresholdBoostFactor); } return annotations.toString(); } @@ -963,8 +947,7 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - serializeWeightedSetContents(destination, WEIGHTED_SET, - (WeightedSetItem) item); + serializeWeightedSetContents(destination, WEIGHTED_SET, (WeightedSetItem) item); return false; } @@ -981,14 +964,12 @@ public class VespaSerializer { WordItem w = (WordItem) item; StringBuilder wordAnnotations = getAllAnnotations(w); - destination.append(normalizeIndexName(w.getIndexName())).append( - " contains "); + destination.append(normalizeIndexName(w.getIndexName())).append(" contains "); VespaSerializer.annotatedTerm(destination, w, wordAnnotations.toString()); return false; } - static void serializeWordWithoutIndex(StringBuilder destination, - Item item) { + static void serializeWordWithoutIndex(StringBuilder destination, Item item) { WordItem w = (WordItem) item; StringBuilder wordAnnotations = getAllAnnotations(w); @@ -996,8 +977,7 @@ public class VespaSerializer { } private static StringBuilder getAllAnnotations(WordItem w) { - StringBuilder wordAnnotations = new StringBuilder( - WordSerializer.wordAnnotations(w)); + StringBuilder wordAnnotations = new StringBuilder(WordSerializer.wordAnnotations(w)); String leafAnnotations = leafAnnotations(w); if (leafAnnotations.length() > 0) { @@ -1034,15 +1014,12 @@ public class VespaSerializer { length = origin.end - origin.start; } - if (!image.substring(offset, offset + length).equals( - item.getIndexedString())) { - VespaSerializer.serializeOrigin(annotation, image, offset, - length); + if (!image.substring(offset, offset + length).equals(item.getIndexedString())) { + VespaSerializer.serializeOrigin(annotation, image, offset, length); } if (usePositionData != true) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(USE_POSITION_DATA) - .append("\": false"); + annotation.append('"').append(USE_POSITION_DATA).append("\": false"); } if (stemmed == true) { VespaSerializer.comma(annotation, initLen); @@ -1050,8 +1027,7 @@ public class VespaSerializer { } if (lowercased == true) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(NORMALIZE_CASE) - .append("\": false"); + annotation.append('"').append(NORMALIZE_CASE).append("\": false"); } if (accentDrop == false) { VespaSerializer.comma(annotation, initLen); @@ -1059,13 +1035,11 @@ public class VespaSerializer { } if (andSegmenting == SegmentingRule.BOOLEAN_AND) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(AND_SEGMENTING) - .append("\": true"); + annotation.append('"').append(AND_SEGMENTING).append("\": true"); } if (!isFromQuery) { VespaSerializer.comma(annotation, initLen); - annotation.append('"').append(IMPLICIT_TRANSFORMS) - .append("\": false"); + annotation.append('"').append(IMPLICIT_TRANSFORMS).append("\": false"); } if (prefix) { VespaSerializer.comma(annotation, initLen); @@ -1106,9 +1080,9 @@ public class VespaSerializer { dispatchBuilder.put(ONearItem.class, new ONearSerializer()); dispatchBuilder.put(OrItem.class, new OrSerializer()); dispatchBuilder.put(PhraseItem.class, new PhraseSerializer()); + dispatchBuilder.put(SameElementItem.class, new SameElementSerializer()); dispatchBuilder.put(PhraseSegmentItem.class, new PhraseSegmentSerializer()); - dispatchBuilder.put(PredicateQueryItem.class, - new PredicateQuerySerializer()); + dispatchBuilder.put(PredicateQueryItem.class, new PredicateQuerySerializer()); dispatchBuilder.put(PrefixItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(WordAlternativesItem.class, new WordAlternativesSerializer()); dispatchBuilder.put(RangeItem.class, new RangeSerializer()); @@ -1225,24 +1199,20 @@ public class VespaSerializer { return out.toString(); } - private static void serializeWeightedSetContents(StringBuilder destination, - String opName, WeightedSetItem weightedSet) { + private static void serializeWeightedSetContents(StringBuilder destination, String opName, + WeightedSetItem weightedSet) { serializeWeightedSetContents(destination, opName, weightedSet, ""); } - private static void serializeWeightedSetContents( - StringBuilder destination, - String opName, WeightedSetItem weightedSet, - String optionalAnnotations) { + private static void serializeWeightedSetContents(StringBuilder destination, String opName, + WeightedSetItem weightedSet, String optionalAnnotations) { addAnnotations(destination, weightedSet, optionalAnnotations); destination.append(opName).append('(') .append(normalizeIndexName(weightedSet.getIndexName())) .append(", {"); int initLen = destination.length(); - List<Entry<Object, Integer>> tokens = new ArrayList<>( - weightedSet.getNumTokens()); - for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i - .hasNext();) { + List<Entry<Object, Integer>> tokens = new ArrayList<>(weightedSet.getNumTokens()); + for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i.hasNext();) { tokens.add(i.next()); } Collections.sort(tokens, tokenComparator); @@ -1255,9 +1225,8 @@ public class VespaSerializer { destination.append("})"); } - private static void addAnnotations( - StringBuilder destination, - WeightedSetItem weightedSet, String optionalAnnotations) { + private static void addAnnotations(StringBuilder destination, WeightedSetItem weightedSet, + String optionalAnnotations) { int preAnnotationValueLen; int incomingLen = destination.length(); String annotations = leafAnnotations(weightedSet); @@ -1303,13 +1272,11 @@ public class VespaSerializer { } if (item.hasExplicitSignificance()) { comma(annotation, initLen); - annotation.append('"').append(SIGNIFICANCE).append("\": ") - .append(significance); + annotation.append('"').append(SIGNIFICANCE).append("\": ").append(significance); } if (uniqueId != 0) { comma(annotation, initLen); - annotation.append('"').append(UNIQUE_ID).append("\": ") - .append(uniqueId); + annotation.append('"').append(UNIQUE_ID).append("\": ").append(uniqueId); } } { @@ -1335,25 +1302,21 @@ public class VespaSerializer { } if (weight != 100) { comma(annotation, initLen); - annotation.append('"').append(WEIGHT).append("\": ") - .append(weight); + annotation.append('"').append(WEIGHT).append("\": ").append(weight); } } if (item instanceof IntItem) { int hitLimit = ((IntItem) item).getHitLimit(); if (hitLimit != 0) { comma(annotation, initLen); - annotation.append('"').append(HIT_LIMIT).append("\": ") - .append(hitLimit); + annotation.append('"').append(HIT_LIMIT).append("\": ").append(hitLimit); } } return annotation.toString(); } - private static void serializeOrigin(StringBuilder destination, - String image, int offset, int length) { - destination.append('"').append(ORIGIN).append("\": {\"") - .append(ORIGIN_ORIGINAL).append("\": \""); + private static void serializeOrigin(StringBuilder destination, String image, int offset, int length) { + destination.append('"').append(ORIGIN).append("\": {\"").append(ORIGIN_ORIGINAL).append("\": \""); escape(image, destination); destination.append("\", \"").append(ORIGIN_OFFSET).append("\": ") .append(offset).append(", \"").append(ORIGIN_LENGTH) diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 259719571be..0b9f79537d0 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -47,6 +47,7 @@ import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentItem; import com.yahoo.prelude.query.SegmentingRule; import com.yahoo.prelude.query.Substring; @@ -101,6 +102,10 @@ public class YqlParser implements Parser { NEVER, POSSIBLY, ALWAYS; } + private static class IndexNameExpander { + public String expand(String leaf) { return leaf; } + } + private static final Integer DEFAULT_HITS = 10; private static final Integer DEFAULT_OFFSET = 0; private static final Integer DEFAULT_TARGET_NUM_HITS = 10; @@ -161,6 +166,7 @@ public class YqlParser implements Parser { static final String RANGE = "range"; static final String RANKED = "ranked"; static final String RANK = "rank"; + static final String SAME_ELEMENT = "sameElement"; static final String SCORE_THRESHOLD = "scoreThreshold"; static final String SIGNIFICANCE = "significance"; static final String STEM = "stem"; @@ -192,6 +198,7 @@ public class YqlParser implements Parser { private Query userQuery; private Parsable currentlyParsing; private IndexFacts.Session indexFactsSession; + private IndexNameExpander indexNameExpander = new IndexNameExpander(); private Set<String> docTypes; private Sorting sorting; private String segmenterBackend; @@ -532,6 +539,31 @@ public class YqlParser implements Parser { return leafStyleSettings(ast, out); } + private static class PrefixExpander extends IndexNameExpander { + private final String prefix; + public PrefixExpander(String prefix) { + this.prefix = prefix + "."; + } + + @Override + public String expand(String leaf) { + return prefix + leaf; + } + } + @NonNull + private Item instantiateSameElementItem(String field, OperatorNode<ExpressionOperator> ast) { + assertHasFunctionName(ast, SAME_ELEMENT); + + SameElementItem sameElement = new SameElementItem(field); + // All terms below sameElement are relative to this. + IndexNameExpander prev = swapIndexCreator(new PrefixExpander(field)); + for (OperatorNode<ExpressionOperator> term : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) { + sameElement.addItem(convertExpression(term)); + } + swapIndexCreator(prev); + return sameElement; + } + @NonNull private Item instantiatePhraseItem(String field, OperatorNode<ExpressionOperator> ast) { assertHasFunctionName(ast, PHRASE); @@ -883,8 +915,16 @@ public class YqlParser implements Parser { @NonNull private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) { - assertHasOperator(ast, ExpressionOperator.READ_FIELD); - return ast.getArgument(1); + switch (ast.getOperator()) { + case READ_FIELD: + return ast.getArgument(1); + case PROPREF: + return new StringBuilder(fetchFieldRead(ast.getArgument(0))) + .append('.').append(ast.getArgument(1).toString()).toString(); + default: + throw newUnexpectedArgumentException(ast.getOperator(), + ExpressionOperator.READ_FIELD, ExpressionOperator.PROPREF); + } } @NonNull @@ -954,14 +994,12 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> lhs = ast.getArgument(0); OperatorNode<ExpressionOperator> rhs = ast.getArgument(1); if (lhs.getOperator() == ExpressionOperator.LITERAL || lhs.getOperator() == ExpressionOperator.NEGATE) { - assertHasOperator(rhs, ExpressionOperator.READ_FIELD); return getIndex(rhs); } if (rhs.getOperator() == ExpressionOperator.LITERAL || rhs.getOperator() == ExpressionOperator.NEGATE) { - assertHasOperator(lhs, ExpressionOperator.READ_FIELD); return getIndex(lhs); } - throw new IllegalArgumentException("Expected LITERAL and READ_FIELD, got " + lhs.getOperator() + + throw new IllegalArgumentException("Expected LITERAL and READ_FIELD/PROPREF, got " + lhs.getOperator() + " and " + rhs.getOperator() + "."); } @@ -977,28 +1015,24 @@ public class YqlParser implements Parser { } @NonNull - private static String fetchConditionWord( - OperatorNode<ExpressionOperator> ast) { + private static String fetchConditionWord(OperatorNode<ExpressionOperator> ast) { OperatorNode<ExpressionOperator> lhs = ast.getArgument(0); OperatorNode<ExpressionOperator> rhs = ast.getArgument(1); - if (lhs.getOperator() == ExpressionOperator.LITERAL - || lhs.getOperator() == ExpressionOperator.NEGATE) { - assertHasOperator(rhs, ExpressionOperator.READ_FIELD); + if (lhs.getOperator() == ExpressionOperator.LITERAL || lhs.getOperator() == ExpressionOperator.NEGATE) { + assertFieldName(rhs); return getNumberAsString(lhs); } - if (rhs.getOperator() == ExpressionOperator.LITERAL - || rhs.getOperator() == ExpressionOperator.NEGATE) { - assertHasOperator(lhs, ExpressionOperator.READ_FIELD); + if (rhs.getOperator() == ExpressionOperator.LITERAL || rhs.getOperator() == ExpressionOperator.NEGATE) { + assertFieldName(lhs); return getNumberAsString(rhs); } - throw new IllegalArgumentException( - "Expected LITERAL/NEGATE and READ_FIELD, got " + throw new IllegalArgumentException("Expected LITERAL/NEGATE and READ_FIELD/PROPREF, got " + lhs.getOperator() + " and " + rhs.getOperator() + "."); } - private static boolean isIndexOnLeftHandSide( - OperatorNode<ExpressionOperator> ast) { - return ast.getArgument(0, OperatorNode.class).getOperator() == ExpressionOperator.READ_FIELD; + private static boolean isIndexOnLeftHandSide(OperatorNode<ExpressionOperator> ast) { + OperatorNode node = ast.getArgument(0, OperatorNode.class); + return node.getOperator() == ExpressionOperator.READ_FIELD || node.getOperator() == ExpressionOperator.PROPREF; } @NonNull @@ -1183,6 +1217,8 @@ public class YqlParser implements Parser { List<String> names = ast.getArgument(0); Preconditions.checkArgument(names.size() == 1, "Expected 1 name, got %s.", names.size()); switch (names.get(0)) { + case SAME_ELEMENT: + return instantiateSameElementItem(field, ast); case PHRASE: return instantiatePhraseItem(field, ast); case NEAR: @@ -1194,7 +1230,7 @@ public class YqlParser implements Parser { case ALTERNATIVES: return instantiateWordAlternativesItem(field, ast); default: - throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE); + throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); } } @@ -1524,6 +1560,12 @@ public class YqlParser implements Parser { expectedFunctionName, names.get(0)); } + private static void assertFieldName(OperatorNode<?> ast) { + Preconditions.checkArgument(ast.getOperator() == ExpressionOperator.READ_FIELD || + ast.getOperator() == ExpressionOperator.PROPREF, + "Expected operator READ_FIELD or PRPPREF, got %s.", ast.getOperator()); + } + private static void addItems(OperatorNode<ExpressionOperator> ast, WeightedSetItem out) { switch (ast.getOperator()) { case MAP: @@ -1604,21 +1646,25 @@ public class YqlParser implements Parser { } } + private IndexNameExpander swapIndexCreator(IndexNameExpander newExpander) { + IndexNameExpander old = indexNameExpander; + indexNameExpander = newExpander; + return old; + } @NonNull private String getIndex(OperatorNode<ExpressionOperator> operatorNode) { String index = fetchFieldRead(operatorNode); - Preconditions.checkArgument(indexFactsSession.isIndex(index), "Field '%s' does not exist.", index); + String expanded = indexNameExpander.expand(index); + Preconditions.checkArgument(indexFactsSession.isIndex(expanded), "Field '%s' does not exist.", expanded); return indexFactsSession.getCanonicName(index); } private Substring getOrigin(OperatorNode<ExpressionOperator> ast) { - Map<?, ?> origin = getAnnotation(ast, ORIGIN, Map.class, null, - ORIGIN_DESCRIPTION); + Map<?, ?> origin = getAnnotation(ast, ORIGIN, Map.class, null, ORIGIN_DESCRIPTION); if (origin == null) { return null; } - String original = getMapValue(ORIGIN, origin, ORIGIN_ORIGINAL, - String.class); + String original = getMapValue(ORIGIN, origin, ORIGIN_ORIGINAL, String.class); int offset = getMapValue(ORIGIN, origin, ORIGIN_OFFSET, Integer.class); int length = getMapValue(ORIGIN, origin, ORIGIN_LENGTH, Integer.class); return new Substring(offset, length + offset, original); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index a9eb9c5e6ce..421f70c8d07 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -141,9 +141,9 @@ public class SlimeSummaryTestCase { @Test public void testFieldAccessAPI() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); + DocsumDefinitionSet fullDocsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); Map<String, Object> expected = new HashMap<>(); @@ -261,6 +261,18 @@ public class SlimeSummaryTestCase { fieldIterator.remove(); expected.remove("integer_field"); assertFields(expected, hit); + + // --- Add full summary + assertNull(fullDocsum.lazyDecode("default", fullishSummary(), hit)); + expected.put("integer_field", 4); + expected.put("short_field", (short)2); + expected.put("byte_field", (byte)1); + expected.put("float_field", 4.5f); + expected.put("double_field", 8.75d); + expected.put("int64_field", 8L); + expected.put("string_field", "string_value"); + expected.put("longstring_field", "longstring_value"); + assertFields(expected, hit); } @@ -274,11 +286,16 @@ public class SlimeSummaryTestCase { traversed.put(name, value); }); assertEquals(expected, traversed); + // raw utf8 field traverser + Map<String, Object> traversedUtf8 = new HashMap<>(); + hit.forEachFieldAsRaw(new Utf8FieldTraverser(traversedUtf8)); + assertEquals(expected, traversedUtf8); // fieldKeys int fieldNameIteratorFieldCount = 0; for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) { fieldNameIteratorFieldCount++; - assertTrue(expected.containsKey(i.next())); + String name = i.next(); + assertTrue("Expected field " + name, expected.containsKey(name)); } assertEquals(expected.size(), fieldNameIteratorFieldCount); // fieldKeys @@ -304,7 +321,7 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte [] timeoutSummary() { + private byte[] timeoutSummary() { Slime slime = new Slime(); slime.setString("Timed out...."); return encode((slime)); @@ -327,6 +344,22 @@ public class SlimeSummaryTestCase { return encode((slime)); } + private byte[] fullishSummary() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + docsum.setLong("byte_field", 1); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + docsum.setLong("int64_field", 8); + docsum.setString("string_field", "string_value"); + //docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("longstring_field", "longstring_value"); + //docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); + return encode((slime)); + } + private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); @@ -346,8 +379,10 @@ public class SlimeSummaryTestCase { field.setLong("foo", 1); field.setLong("bar", 2); } - docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); - docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); + if (tensor1 != null) + docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); + if (tensor2 != null) + docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); return encode((slime)); } @@ -371,4 +406,26 @@ public class SlimeSummaryTestCase { return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); } + private static class Utf8FieldTraverser implements Hit.RawUtf8Consumer { + + private Map<String, Object> traversed; + + public Utf8FieldTraverser(Map<String, Object> traversed) { + this.traversed = traversed; + } + + @Override + public void accept(String fieldName, byte[] utf8Data, int offset, int length) { + traversed.put(fieldName, new String(utf8Data, offset, length, StandardCharsets.UTF_8)); + } + + @Override + public void accept(String name, Object value) { + if (name.equals("string_value")) + fail("Expected string_value to be received as UTF-8"); + traversed.put(name, value); + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java new file mode 100644 index 00000000000..0e4b994fe99 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/SameElementItemTestCase.java @@ -0,0 +1,58 @@ +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class SameElementItemTestCase { + @Test + public void testAddItem() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "f2")); + s.addItem(new WordItem("d", "f3")); + assertEquals("structa:{f1:b f2:c f3:d}", s.toString()); + } + @Test + public void testClone() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "f2")); + s.addItem(new WordItem("d", "f3")); + assertEquals("structa:{f1:b f2:c f3:d}", s.toString()); + SameElementItem c = (SameElementItem)s.clone(); + assertEquals("structa:{f1:b f2:c f3:d}", c.toString()); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenHaveStructMemberNameSet() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c")); + } + @Test + public void requireAllowCommonPrefix() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "structaf2")); + assertEquals("structa:{f1:b structaf2:c}", s.toString()); + } + @Test(expected = IllegalArgumentException.class) + public void requireNoChildrenHasCommonPrefixWithDot() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("b", "f1")); + s.addItem(new WordItem("c", "structa.f2")); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenHaveNonEmptyTerm() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new WordItem("", "f2")); + } + @Test(expected = IllegalArgumentException.class) + public void requireAllChildrenAreTermItems() { + SameElementItem s = new SameElementItem("structa"); + s.addItem(new AndItem()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java index 091e2c1772d..7858bbf6433 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java @@ -14,6 +14,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.test.QueryTestCase; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 4987f9902ce..fae869c5235 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -17,16 +17,15 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.searcher.BlendingSearcher; -import com.yahoo.prelude.searcher.DocumentSourceSearcher; import com.yahoo.prelude.searcher.FillSearcher; import com.yahoo.search.Searcher; import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.federation.selection.TargetSelector; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChain; import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -40,8 +39,6 @@ import static org.junit.Assert.assertTrue; * @author Bob Travis * @author bratseth */ -// The SuppressWarnings is to shut up the compiler about using -// deprecated FastHit constructor in the tests. @SuppressWarnings({ "rawtypes" }) public class BlendingSearcherTestCase { @@ -139,19 +136,19 @@ public class BlendingSearcherTestCase { r1.hits().add(new Hit("http://host1.com", 101){{setSource("one");}}); r1.hits().add(new Hit("http://host2.com", 102){{setSource("one");}}); r1.hits().add(new Hit("http://host3.com", 103){{setSource("one");}}); - chain1.addResultSet(q, r1); + chain1.addResult(q, r1); r2.setTotalHitCount(17); r2.hits().add(new Hit("http://host1.com", 101){{setSource("two");}}); r2.hits().add(new Hit("http://host2.com", 102){{setSource("two");}}); r2.hits().add(new Hit("http://host4.com", 104){{setSource("two");}}); - chain2.addResultSet(q, r2); + chain2.addResult(q, r2); r3.setTotalHitCount(37); r3.hits().add(new Hit("http://host5.com", 100){{setSource("three");}}); r3.hits().add(new Hit("http://host6.com", 106){{setSource("three");}}); r3.hits().add(new Hit("http://host7.com", 105){{setSource("three");}}); - chain3.addResultSet(q, r3); + chain3.addResult(q, r3); BlendingSearcherWrapper blender1 = new BlendingSearcherWrapper(); blender1.addChained(chain1, "one"); @@ -215,10 +212,10 @@ public class BlendingSearcherTestCase { r1.setTotalHitCount(1); r1.hits().add(new FastHit("http://host1.com/", 101)); - chain1.addResultSet(q, r1); + chain1.addResult(q, r1); r2.hits().add(new FastHit("http://host1.com/", 102)); r2.setTotalHitCount(1); - chain2.addResultSet(q, r2); + chain2.addResult(q, r2); BlendingSearcherWrapper blender = new BlendingSearcherWrapper("uri"); blender.addChained(new FillSearcher(chain1), "a"); @@ -239,10 +236,10 @@ public class BlendingSearcherTestCase { Result r2 = new Result(q, ErrorMessage.createRequestTooLarge(null)); r1.setTotalHitCount(0); - chain1.addResultSet(q, r1); + chain1.addResult(q, r1); r2.hits().add(new FastHit("http://host1.com/", 102)); r2.setTotalHitCount(1); - chain2.addResultSet(q, r2); + chain2.addResult(q, r2); BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); blender.addChained(new FillSearcher(chain1), "a"); @@ -288,7 +285,7 @@ public class BlendingSearcherTestCase { r1.hits().add(r1h1); r1.hits().add(r1h2); r1.hits().add(r1h3); - chain1.addResultSet(q, r1); + chain1.addResult(q, r1); r2.setTotalHitCount(3); Hit r2h1 = new Hit("http://host1.com/relevancy201", 201); @@ -303,7 +300,7 @@ public class BlendingSearcherTestCase { r2.hits().add(r2h1); r2.hits().add(r2h2); r2.hits().add(r2h3); - chain2.addResultSet(q, r2); + chain2.addResult(q, r2); BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); blender.addChained(new FillSearcher(chain1), "chainedone"); @@ -343,7 +340,7 @@ public class BlendingSearcherTestCase { r1.hits().add(r1h1); r1.hits().add(r1h2); r1.hits().add(r1h3); - chain1.addResultSet(q, r1); + chain1.addResult(q, r1); r2.setTotalHitCount(3); Hit r2h1 = new Hit("http://host1.com/relevancy201", 201); @@ -355,7 +352,7 @@ public class BlendingSearcherTestCase { r2.hits().add(r2h1); r2.hits().add(r2h2); r2.hits().add(r2h3); - chain2.addResultSet(q, r2); + chain2.addResult(q, r2); BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); blender.addChained(chain1, "chainedone"); @@ -381,7 +378,7 @@ public class BlendingSearcherTestCase { r1.setTotalHitCount(1); Hit r1h1 = new Hit("http://first/relevancy100", 200); r1.hits().add(r1h1); - first.addResultSet(query, r1); + first.addResult(query, r1); Result r2 = new Result(query); r2.setTotalHitCount(2); @@ -389,7 +386,7 @@ public class BlendingSearcherTestCase { Hit r2h2 = new Hit("http://second/relevancy100", 100); r2.hits().add(r2h1); r2.hits().add(r2h2); - second.addResultSet(query, r2); + second.addResult(query, r2); BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); blender.addChained(new FillSearcher(first), "first"); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index f71c0803ce8..9f4a12d24e6 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -28,9 +28,9 @@ import java.util.List; import java.util.Map; /** - * Tests conversion of juniper highlighting to XML + * Tests juniper highlighting * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JuniperSearcherTestCase { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index ee735104caa..691f877dfba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -15,9 +15,9 @@ import com.yahoo.search.Result; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.HitField; import com.yahoo.search.Searcher; -import com.yahoo.prelude.searcher.DocumentSourceSearcher; import com.yahoo.prelude.searcher.QuotingSearcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import java.util.ArrayList; @@ -55,7 +55,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "smith & jones"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("smith & jones", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -75,7 +75,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "&smith &jo& nes"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -95,7 +95,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", new HitField("title", "&smith &jo& nes")); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -116,7 +116,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", Integer.valueOf(42)); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); // should not quote non-string properties assertEquals(Integer.valueOf(42), check.hits().get(0).getField("title")); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 99b7eeff9a0..a3f7ff12319 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -46,32 +46,32 @@ public class NGramSearcherTestCase { @Before public void setUp() { - searcher=new NGramSearcher(new SimpleLinguistics()); - indexFacts=new IndexFacts(); + searcher = new NGramSearcher(new SimpleLinguistics()); + indexFacts = new IndexFacts(); - Index defaultIndex=new Index("default"); - defaultIndex.setNGram(true,3); + Index defaultIndex = new Index("default"); + defaultIndex.setNGram(true, 3); defaultIndex.setDynamicSummary(true); - indexFacts.addIndex("default",defaultIndex); + indexFacts.addIndex("default", defaultIndex); - Index test=new Index("test"); + Index test = new Index("test"); test.setHighlightSummary(true); - indexFacts.addIndex("default",test); + indexFacts.addIndex("default", test); - Index gram2=new Index("gram2"); - gram2.setNGram(true,2); + Index gram2 = new Index("gram2"); + gram2.setNGram(true, 2); gram2.setDynamicSummary(true); - indexFacts.addIndex("default",gram2); + indexFacts.addIndex("default", gram2); - Index gram3=new Index("gram3"); - gram3.setNGram(true,3); + Index gram3 = new Index("gram3"); + gram3.setNGram(true, 3); gram3.setHighlightSummary(true); - indexFacts.addIndex("default",gram3); + indexFacts.addIndex("default", gram3); - Index gram14=new Index("gram14"); - gram14.setNGram(true,14); + Index gram14 = new Index("gram14"); + gram14.setNGram(true, 14); gram14.setDynamicSummary(true); - indexFacts.addIndex("default",gram14); + indexFacts.addIndex("default", gram14); } private IndexFacts getMixedSetup() { diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 4a0075c0cf1..bf56ad19f44 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -101,7 +101,7 @@ public class JsonRendererTestCase { } @Test - public void testDocumentId() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testDocumentId() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index d6bcdf3195f..6984a8537ef 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.search.yql; import static org.junit.Assert.*; +import com.yahoo.prelude.query.SameElementItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; @@ -218,6 +219,15 @@ public class VespaSerializerTestCase { } @Test + public final void testSameElement() { + SameElementItem sameElement = new SameElementItem("ss"); + sameElement.addItem(new WordItem("a", "f1")); + sameElement.addItem(new WordItem("b", "f2")); + assertEquals("ss:{f1:a f2:b}", sameElement.toString()); + assertEquals("ss contains sameElement(f1 contains ([{\"implicitTransforms\": false}]\"a\"), f2 contains ([{\"implicitTransforms\": false}]\"b\"))", VespaSerializer.serialize(sameElement)); + + } + @Test public final void testAnnotatedAndSegment() { AndSegmentItem andSegment = new AndSegmentItem("abc", true, false); andSegment.addItem(new WordItem("a", "indexNamePlaceholder")); @@ -307,6 +317,7 @@ public class VespaSerializerTestCase { assertEquals("default contains \"nalle\"", q); } + @Test public final void testLongAndNot() { NotItem item = new NotItem(); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 75b39f26625..8bbe43ee3d4 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -118,6 +118,17 @@ public class YqlParserTestCase { } @Test + public void testDottedFieldNames() { + assertParse("select foo from bar where my.nested.title contains \"madonna\";", + "my.nested.title:madonna"); + } + @Test + public void testDottedNestedFieldNames() { + assertParse("select foo from bar where my.title contains \"madonna\";", + "my.title:madonna"); + } + + @Test public void testOr() { assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\";", "OR title:madonna title:saint"); @@ -261,6 +272,16 @@ public class YqlParserTestCase { } @Test + public void testSameElement() { + assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 contains \"b\");", + "baz:{f1:a f2:b}"); + assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 = 10);", + "baz:{f1:a f2:10}"); + assertParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10);", + "baz:{key:a value.f2:10}"); + } + + @Test public void testPhrase() { assertParse("select foo from bar where baz contains phrase(\"a\", \"b\");", "baz:\"a b\""); diff --git a/container-test-jars/jersey-resources/pom.xml b/container-test-jars/jersey-resources/pom.xml index 33a3f03a962..2b6761e6411 100644 --- a/container-test-jars/jersey-resources/pom.xml +++ b/container-test-jars/jersey-resources/pom.xml @@ -16,11 +16,6 @@ <dependencies> <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespa_jersey2</artifactId> <version>${project.version}</version> @@ -31,27 +26,6 @@ <build> <plugins> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>add-source</goal> - <goal>compile</goal> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - <configuration> - <args> - <arg>-unchecked</arg> - <arg>-deprecation</arg> - <arg>-feature</arg> - </args> - </configuration> - </plugin> - - <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java new file mode 100644 index 00000000000..59095d05567 --- /dev/null +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResource.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources; + +import javax.ws.rs.Path; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +@Path("bundle-plugin-test/test-resource") +public class TestResource extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java new file mode 100644 index 00000000000..c3724723252 --- /dev/null +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.java @@ -0,0 +1,22 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.Produces; +import javax.ws.rs.GET; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class TestResourceBase { + @GET + @Produces({MediaType.TEXT_PLAIN}) + public String get() { + return content(getClass()); + } + + public static String content(Class<? extends TestResourceBase> clazz) { + return "Response from " + clazz.getName(); + } +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.scala b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java index 440f2f45cea..ab1c1f8f229 100644 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.scala +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage1/NestedTestResource1.java @@ -1,12 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage1 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources.nestedpackage1; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes */ @Path("bundle-plugin-test/nested-test-resource1") -class NestedTestResource1 extends TestResourceBase +public class NestedTestResource1 extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.scala b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java index 5fa354dd647..0dfc9e1938b 100644 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.scala +++ b/container-test-jars/jersey-resources/src/main/java/com/yahoo/container/test/jars/jersey/resources/nestedpackage2/NestedTestResource2.java @@ -1,12 +1,13 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources.nestedpackage2 +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test.jars.jersey.resources.nestedpackage2; -import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase; -import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import javax.ws.rs.Path; /** - * @author tonytv + * @author Tony Vaagenes */ @Path("bundle-plugin-test/nested-test-resource2") -class NestedTestResource2 extends TestResourceBase +public class NestedTestResource2 extends TestResourceBase { +} diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala b/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala deleted file mode 100644 index b73a8735789..00000000000 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResource.scala +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources - -import javax.ws.rs.Path - -/** - * @author tonytv - */ -@Path("bundle-plugin-test/test-resource") -class TestResource extends TestResourceBase diff --git a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala b/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala deleted file mode 100644 index 5ccd89b30ac..00000000000 --- a/container-test-jars/jersey-resources/src/main/scala/com/yahoo/container/test/jars/jersey/resources/TestResourceBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.test.jars.jersey.resources - -import javax.ws.rs.core.MediaType -import javax.ws.rs.{Produces, GET} - -import scala.reflect.ClassTag - -/** - * @author tonytv - */ -class TestResourceBase { - @GET - @Produces(Array(MediaType.TEXT_PLAIN)) - def get() = TestResourceBase.content(getClass) -} - -object TestResourceBase { - def content(clazz: Class[_ <: TestResourceBase]): String = - "Response from " + clazz.getName - - def content[T <: TestResourceBase](implicit classTag: ClassTag[T]): String = { - val clazz = classTag.runtimeClass.asInstanceOf[Class[_ <: TestResourceBase]] - content(clazz) - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index a2b420c8090..b15acd726d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -38,8 +37,6 @@ public interface ConfigServer { void deactivate(DeploymentId applicationInstance) throws NoInstanceException; - JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds); - ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index db749713483..8b0dc35e16b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -341,16 +341,16 @@ public class ApplicationController { /** Deploy a system application to given zone */ public void deploy(SystemApplication application, ZoneId zone, Version version) { - if (!application.hasApplicationPackage()) { + if (application.hasApplicationPackage()) { + ApplicationPackage applicationPackage = new ApplicationPackage( + artifactRepository.getSystemApplicationPackage(application.id(), zone, version) + ); + DeployOptions options = withVersion(version, DeployOptions.none()); + deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + } else { // Deploy by calling node repository directly configServer().nodeRepository().upgrade(zone, application.nodeType(), version); - return; } - ApplicationPackage applicationPackage = new ApplicationPackage( - artifactRepository.getSystemApplicationPackage(application.id(), zone, version) - ); - DeployOptions options = withVersion(version, DeployOptions.none()); - deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 8fb37ba2a15..b65d3bc0849 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; @@ -10,7 +9,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; @@ -157,11 +155,6 @@ public class Controller extends AbstractComponent { return globalRoutingService.getHealthStatus(rotation.name()); } - // TODO: Model the response properly - public JsonNode waitForConfigConvergence(DeploymentId deploymentId, long timeout) { - return configServer.waitForConfigConverge(deploymentId, timeout); - } - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { return configServer.getApplicationView(tenantName, applicationName, instanceName, environment, region); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index fb8a4051649..d3193fd486d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -137,7 +137,7 @@ public class DeploymentJobs { productionApSoutheast1 ("production-ap-southeast-1" , ZoneId.from("prod" , "ap-southeast-1") , null ), productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ), - productionCdUsEast1a ("production-cd-us-east-1a" , null , ZoneId.from("prod" , "cd-us-east-1a") ), + productionCdAwsUsEast1a("production-cd-aws-us-east-1a" , null , ZoneId.from("prod" , "cd-aws-us-east-1a")), productionCdUsCentral1 ("production-cd-us-central-1", null , ZoneId.from("prod" , "cd-us-central-1")), productionCdUsCentral2 ("production-cd-us-central-2", null , ZoneId.from("prod" , "cd-us-central-2")); @@ -180,10 +180,15 @@ public class DeploymentJobs { return zone(system).map(ZoneId::region); } - public static JobType fromJobName(String jobName) { + public static Optional<JobType> fromOptionalJobName(String jobName) { return Stream.of(values()) .filter(jobType -> jobType.jobName.equals(jobName)) - .findAny().orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); + .findAny(); + } + + public static JobType fromJobName(String jobName) { + return fromOptionalJobName(jobName) + .orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); } /** Returns the job type for the given zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index d3daa68741e..1e96f33c275 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -12,17 +12,23 @@ import java.util.List; * * @author mpolden */ -public enum SystemApplication { +public enum SystemApplication { + // Note that the enum declaration order decides the upgrade order + configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), + proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy); + zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, + configServerHost, proxyHost, configServer); private final ApplicationId id; private final NodeType nodeType; + private final List<SystemApplication> dependencies; - SystemApplication(ApplicationId id, NodeType nodeType) { + SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { this.id = id; this.nodeType = nodeType; + this.dependencies = Arrays.asList(dependencies); } public ApplicationId id() { @@ -34,7 +40,10 @@ public enum SystemApplication { return nodeType; } - /** Returns whether this system application has its own application package */ + /** Returns the system applications that should upgrade before this */ + public List<SystemApplication> dependencies() { return dependencies; } + + /** Returns whether this system application has an application package */ public boolean hasApplicationPackage() { return nodeType == NodeType.proxy; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 75c01dcb9b3..e902206ad8b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -97,7 +97,7 @@ public class DeploymentTrigger { * trigger next. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d).", + log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d)", report.jobError().map(e -> e.toString() + " error") .orElse("success"), report.jobType(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 93103213274..7154cf7d600 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -58,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); - systemUpgrader = new SystemUpgrader(controller, maintenanceInterval, jobControl); + systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index d7fc71e4b08..ac0d08f5105 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -37,51 +37,78 @@ public class SystemUpgrader extends Maintainer { if (!target.isPresent()) { return; } + + deploy(SystemApplication.all(), target.get()); + } + + /** Deploy a list of system applications until they converge on the given version */ + private void deploy(List<SystemApplication> applications, Version target) { for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) { - // The order here is important. Config servers should always upgrade first - if (!deploy(zones, SystemApplication.configServer, target.get())) { - break; + boolean converged = true; + for (ZoneId zone : zones) { + try { + converged &= deployInZone(zone, applications, target); + } catch (UnreachableNodeRepositoryException e) { + converged = false; + log.log(Level.WARNING, e.getMessage() + ". Continuing to next parallel deployed zone"); + } catch (Exception e) { + converged = false; + log.log(Level.WARNING, "Failed to upgrade " + zone + ". Continuing to next parallel deployed zone", e); + } } - if (!deploy(zones, SystemApplication.zone, target.get())) { + if (!converged) { break; } } } - /** Deploy application on given version. Returns true when all allocated nodes are on requested version */ - private boolean deploy(List<ZoneId> zones, SystemApplication application, Version version) { - boolean completed = true; - for (ZoneId zone : zones) { - if (!wantedVersion(zone, application.id()).equals(version)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), version, zone)); - controller().applications().deploy(application, zone, version); + /** @return true if all applications have converged to the target version in the zone */ + private boolean deployInZone(ZoneId zone, List<SystemApplication> applications, Version target) { + boolean converged = true; + for (SystemApplication application : applications) { + if (convergedOn(target, application.dependencies(), zone)) { + deploy(target, application, zone); } - completed = completed && currentVersion(zone, application.id()).equals(version); + converged &= convergedOn(target, application, zone); + } + return converged; + } + + /** Deploy application on given version idempotently */ + private void deploy(Version target, SystemApplication application, ZoneId zone) { + if (!wantedVersion(zone, application.id(), target).equals(target)) { + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); + controller().applications().deploy(application, zone, target); } - return completed; } - private Version wantedVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::wantedVersion); + private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) { + return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } - private Version currentVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::currentVersion); + private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + return currentVersion(zone, application.id(), target).equals(target); } - private Version minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { + private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion); + } + + private Version currentVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + return minVersion(zone, application, Node::currentVersion).orElse(defaultVersion); + } + + private Optional<Version> minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { try { return controller().configServer() .nodeRepository() .listOperational(zone, application) .stream() .map(versionField) - .min(Comparator.naturalOrder()) - .orElse(Version.emptyVersion); + .min(Comparator.naturalOrder()); } catch (Exception e) { - log.log(Level.WARNING, String.format("Failed to get version for %s in %s: %s", application, zone, - Exceptions.toMessageString(e))); - return Version.emptyVersion; + throw new UnreachableNodeRepositoryException(String.format("Failed to get version for %s in %s: %s", + application, zone, Exceptions.toMessageString(e))); } } @@ -92,4 +119,9 @@ public class SystemUpgrader extends Maintainer { .map(VespaVersion::versionNumber); } + private class UnreachableNodeRepositoryException extends RuntimeException { + private UnreachableNodeRepositoryException(String reason) { + super(reason); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 531078b7eca..21eea21ba68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -382,22 +382,25 @@ public class ApplicationSerializer { private List<JobStatus> jobStatusListFromSlime(Inspector array) { List<JobStatus> jobStatusList = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusList.add(jobStatusFromSlime(item))); + array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusFromSlime(item).ifPresent(jobStatusList::add)); return jobStatusList; } - private JobStatus jobStatusFromSlime(Inspector object) { - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromJobName(object.field(jobTypeField).asString()); + private Optional<JobStatus> jobStatusFromSlime(Inspector object) { + // if the job type has since been removed, ignore it + Optional<DeploymentJobs.JobType> jobType = + DeploymentJobs.JobType.fromOptionalJobName(object.field(jobTypeField).asString()); + if (! jobType.isPresent()) return Optional.empty(); Optional<JobError> jobError = Optional.empty(); if (object.field(errorField).valid()) jobError = Optional.of(JobError.valueOf(object.field(errorField).asString())); - return new JobStatus(jobType, jobError, + return Optional.of(new JobStatus(jobType.get(), jobError, jobRunFromSlime(object.field(lastTriggeredField)), jobRunFromSlime(object.field(lastCompletedField)), jobRunFromSlime(object.field(firstFailingField)), - jobRunFromSlime(object.field(lastSuccessField))); + jobRunFromSlime(object.field(lastSuccessField)))); } private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index cbd4614e51b..e5e03cf5dfc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -40,6 +40,7 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -80,16 +81,12 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ZoneId zoneId = ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion()); // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId)); + List<URI> allServers = zoneRegistry.getConfigServerVipUri(zoneId) + // TODO: Use config server VIP for all zones that have one + .filter(zone -> zoneId.region().value().startsWith("aws-") || zoneId.region().value().startsWith("cd-aws-")) - // TODO: Use config server VIP for all zones that have one - - // For now, just add config server VIP as first element in list of servers if it exists - if (zoneId.region().value().startsWith("aws-") - || zoneId.region().value().startsWith("cd-aws-") - || zoneId.region().value().equals("cd-us-east-1a")) { - zoneRegistry.getConfigServerVipUri(zoneId).ifPresent(uri -> allServers.add(0, uri)); - } + .map(Collections::singletonList) + .orElseGet(() -> new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId))); StringBuilder errorBuilder = new StringBuilder(); if (queueFirstServerIfDown(allServers, proxyRequest)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d9848577f0a..10088ba3fea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -168,7 +168,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -579,12 +578,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)), - asLong(request.getProperty("timeout"), 1000))); - } - private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java deleted file mode 100644 index cfd6feccf01..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * @author bratseth - */ -public class JacksonJsonResponse extends HttpResponse { - - private final JsonNode node; - - public JacksonJsonResponse(JsonNode node) { - super(200); - this.node = node; - } - - @Override - public void render(OutputStream stream) throws IOException { - new ObjectMapper().writeValue(stream, node); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java index e9eec0682f3..ac9afd9752a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java @@ -1,9 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; @@ -54,13 +51,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { - bootstrap(zoneRegistry.zones().all().ids()); + bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all()); } - public void bootstrap(List<ZoneId> zones) { + public void bootstrap(List<ZoneId> zones, SystemApplication... applications) { + bootstrap(zones, Arrays.asList(applications)); + } + + public void bootstrap(List<ZoneId> zones, List<SystemApplication> applications) { nodeRepository().clear(); for (ZoneId zone : zones) { - for (SystemApplication application : SystemApplication.all()) { + for (SystemApplication application : applications) { List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() @@ -182,13 +183,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.remove(deployment.applicationId()); } - @Override - public JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds) { - ObjectNode root = new ObjectNode(JsonNodeFactory.instance); - root.put("generation", 1); - return root; - } - // Returns a canned example response @Override public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 9d5fcb31288..a81c4adcb2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -85,7 +85,7 @@ public class ApplicationPackageBuilder { public ApplicationPackageBuilder allow(ValidationId validationId) { validationOverridesBody.append(" <allow until='"); - validationOverridesBody.append(asIso8601Date(Instant.now().plus(Duration.ofDays(29)))); + validationOverridesBody.append(asIso8601Date(Instant.now().plus(Duration.ofDays(28)))); validationOverridesBody.append("'>"); validationOverridesBody.append(validationId.value()); validationOverridesBody.append("</allow>\n"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 5790ab2a0c8..4b563ed203d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; @@ -16,6 +17,7 @@ import java.util.List; import java.util.function.Function; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author mpolden @@ -39,7 +41,9 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4)); + // Bootstrap a system without host applications + tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.configServer, + SystemApplication.zone); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); failNodeIn(zone3, SystemApplication.zone); @@ -68,17 +72,25 @@ public class SystemUpgraderTest { tester.systemUpgrader().maintain(); assertWantedVersion(SystemApplication.zone, version2, zone1); completeUpgrade(SystemApplication.zone, version2, zone1); + assertTrue("Deployed zone application", + tester.configServer().application(SystemApplication.zone.id()).isPresent()); // zone 2, 3 and 4: still targets old version assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4); assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); - // zone 2 and 3: zone-config-server upgrades in parallel + // zone 2 and 3: zone-config-server upgrades, first in zone 2, then in zone 3 tester.systemUpgrader().maintain(); assertWantedVersion(SystemApplication.configServer, version2, zone2, zone3); assertWantedVersion(SystemApplication.configServer, version1, zone4); assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); - completeUpgrade(SystemApplication.configServer, version2, zone2, zone3); + completeUpgrade(SystemApplication.configServer, version2, zone2); + + // zone-application starts upgrading in zone 2, while zone-config-server completes upgrade in zone 3 + tester.systemUpgrader().maintain(); + assertWantedVersion(SystemApplication.zone, version2, zone2); + assertWantedVersion(SystemApplication.zone, version1, zone3); + completeUpgrade(SystemApplication.configServer, version2, zone3); // zone 2 and 3: zone-application upgrades in parallel tester.systemUpgrader().maintain(); @@ -109,6 +121,54 @@ public class SystemUpgraderTest { } @Test + public void upgrade_system_containing_host_applications() { + tester.controllerTester().zoneRegistry().setUpgradePolicy( + UpgradePolicy.create() + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone4) + ); + + Version version1 = Version.fromString("6.5"); + tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4), SystemApplication.all()); + tester.upgradeSystem(version1); + tester.systemUpgrader().maintain(); + assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); + + // Controller upgrades + Version version2 = Version.fromString("6.6"); + tester.upgradeController(version2); + assertEquals(version2, tester.controller().versionStatus().controllerVersion().get().versionNumber()); + + // System upgrades in zone 1: + tester.systemUpgrader().maintain(); + List<SystemApplication> allExceptZone = Arrays.asList(SystemApplication.configServerHost, + SystemApplication.proxyHost, + SystemApplication.configServer); + completeUpgrade(allExceptZone, version2, zone1); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone1); + assertWantedVersion(SystemApplication.all(), version1, zone2, zone3, zone4); + + // zone 2 and 3: + tester.systemUpgrader().maintain(); + completeUpgrade(allExceptZone, version2, zone2, zone3); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone2, zone3); + assertWantedVersion(SystemApplication.all(), version1, zone4); + + // zone 4: + tester.systemUpgrader().maintain(); + completeUpgrade(allExceptZone, version2, zone4); + tester.systemUpgrader().maintain(); + completeUpgrade(SystemApplication.zone, version2, zone4); + + // All done + tester.systemUpgrader().maintain(); + assertWantedVersion(SystemApplication.all(), version2, zone1, zone2, zone3, zone4); + } + + @Test public void never_downgrades_system() { ZoneId zone = ZoneId.from("prod", "eu-west-1"); tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone)); @@ -130,6 +190,7 @@ public class SystemUpgraderTest { /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { + assertWantedVersion(application, version, zones); for (ZoneId zone : zones) { for (Node node : nodeRepository().listOperational(zone, application.id())) { nodeRepository().add(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), @@ -139,6 +200,10 @@ public class SystemUpgraderTest { } } + private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> completeUpgrade(application, version, zones)); + } + private void failNodeIn(ZoneId zone, SystemApplication application) { List<Node> nodes = nodeRepository().list(zone, application.id()); if (nodes.isEmpty()) { @@ -157,11 +222,19 @@ public class SystemUpgraderTest { assertVersion(application.id(), version, Node::currentVersion, zones); } + private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> assertVersion(application.id(), version, Node::wantedVersion, zones)); + } + + private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + applications.forEach(application -> assertVersion(application.id(), version, Node::currentVersion, zones)); + } + private void assertVersion(ApplicationId application, Version version, Function<Node, Version> versionField, ZoneId... zones) { for (ZoneId zone : zones) { for (Node node : nodeRepository().listOperational(zone, application)) { - assertEquals(version, versionField.apply(node)); + assertEquals(application + " version", version, versionField.apply(node)); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 407263a7973..8d734ec549c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -294,14 +293,12 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", POST) .screwdriverIdentity(SCREWDRIVER_ID), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); - // GET (wait for) convergence - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", GET) - .userIdentity(USER_ID), - new File("convergence.json")); + // GET services tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET) .userIdentity(USER_ID), @@ -319,11 +316,13 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE) .userIdentity(USER_ID), "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); + // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // DELETE (deactivate) a deployment is idempotent tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json deleted file mode 100644 index acfb67b702b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "generation": 1 -}
\ No newline at end of file diff --git a/defaults/src/apps/printdefault/CMakeLists.txt b/defaults/src/apps/printdefault/CMakeLists.txt index 03fe38bcef4..2823b90d251 100644 --- a/defaults/src/apps/printdefault/CMakeLists.txt +++ b/defaults/src/apps/printdefault/CMakeLists.txt @@ -7,3 +7,4 @@ vespa_add_executable(defaults_vespa-print-default_app DEPENDS vespadefaults ) +set_source_files_properties(printdefault.cpp PROPERTIES COMPILE_FLAGS "${VTAG_DEFINES}") diff --git a/defaults/src/apps/printdefault/printdefault.cpp b/defaults/src/apps/printdefault/printdefault.cpp index 73e199fb441..33c47dbcb20 100644 --- a/defaults/src/apps/printdefault/printdefault.cpp +++ b/defaults/src/apps/printdefault/printdefault.cpp @@ -8,8 +8,9 @@ int main(int argc, char **argv) { if (argc != 2) { fprintf(stderr, "usage: %s <variable>\n", argv[0]); fprintf(stderr, " variable names are: home, user, hostname, portbase, configservers,\n"); - fprintf(stderr, " configserver_rpc_port, configservers_rpc\n"); - fprintf(stderr, " configservers_http, configsources, configproxy_rpc\n"); + fprintf(stderr, " configserver_rpc_port, configservers_rpc,\n"); + fprintf(stderr, " configservers_http, configsources, configproxy_rpc,\n"); + fprintf(stderr, " version\n"); return 1; } if (strcmp(argv[1], "home") == 0) { @@ -47,6 +48,8 @@ int main(int argc, char **argv) { } else if (strcmp(argv[1], "configproxy_rpc") == 0) { std::string v = vespa::Defaults::vespaConfigProxyRpcAddr(); printf("%s\n", v.c_str()); + } else if (strcmp(argv[1], "version") == 0) { + printf("%s\n", V_TAG_COMPONENT); } else { fprintf(stderr, "Unknown variable '%s'\n", argv[1]); return 1; diff --git a/dist/vespa.spec b/dist/vespa.spec index fc110ba8f4f..0e3cabcea80 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -21,9 +21,9 @@ BuildRequires: centos-release-scl BuildRequires: devtoolset-7-gcc-c++ BuildRequires: devtoolset-7-libatomic-devel BuildRequires: devtoolset-7-binutils -BuildRequires: rh-maven33 +BuildRequires: rh-maven35 %define _devtoolset_enable /opt/rh/devtoolset-7/enable -%define _rhmaven33_enable /opt/rh/rh-maven33/enable +%define _rhmaven35_enable /opt/rh/rh-maven35/enable %endif %if 0%{?fedora} BuildRequires: gcc-c++ @@ -42,18 +42,24 @@ BuildRequires: maven %if 0%{?fc26} BuildRequires: llvm-devel >= 4.0 BuildRequires: boost-devel >= 1.63 +BuildRequires: vespa-gtest >= 1.8.0-2 %endif %if 0%{?fc27} BuildRequires: llvm4.0-devel >= 4.0 BuildRequires: boost-devel >= 1.64 +BuildRequires: vespa-gtest >= 1.8.0-2 %endif %if 0%{?fc28} BuildRequires: llvm4.0-devel >= 4.0 BuildRequires: boost-devel >= 1.66 +BuildRequires: gtest-devel +BuildRequires: gmock-devel %endif %if 0%{?fc29} BuildRequires: llvm4.0-devel >= 4.0 BuildRequires: boost-devel >= 1.66 +BuildRequires: gtest-devel +BuildRequires: gmock-devel %endif %endif BuildRequires: lz4-devel @@ -106,12 +112,16 @@ Requires: llvm3.9 %if 0%{?fc26} Requires: llvm-libs >= 4.0 %define _vespa_llvm_version 4.0 +%define _vespa_gtest_link_directory /opt/vespa-gtest/lib +%define _vespa_gtest_include_directory /opt/vespa-gtest/include %endif %if 0%{?fc27} Requires: llvm4.0-libs >= 4.0 %define _vespa_llvm_version 4.0 %define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib %define _vespa_llvm_include_directory /usr/include/llvm4.0 +%define _vespa_gtest_link_directory /opt/vespa-gtest/lib +%define _vespa_gtest_include_directory /opt/vespa-gtest/include %endif %if 0%{?fc28} Requires: llvm4.0-libs >= 4.0 @@ -125,8 +135,8 @@ Requires: llvm4.0-libs >= 4.0 %define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib %define _vespa_llvm_include_directory /usr/include/llvm4.0 %endif -%define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}} -%define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}} +%define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}}%{?_vespa_gtest_link_directory:;%{_vespa_gtest_link_directory}} +%define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}}%{?_vespa_gtest_include_directory:;%{_vespa_gtest_include_directory}} %endif Requires: java-1.8.0-openjdk Requires: openssl @@ -148,8 +158,8 @@ Vespa - The open big data serving engine %if 0%{?_devtoolset_enable:1} source %{_devtoolset_enable} || true %endif -%if 0%{?_rhmaven33_enable:1} -source %{_rhmaven33_enable} || true +%if 0%{?_rhmaven35_enable:1} +source %{_rhmaven35_enable} || true %endif sh bootstrap.sh java mvn --batch-mode -nsu -T 2C install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true diff --git a/document/src/main/java/com/yahoo/document/DocumentPut.java b/document/src/main/java/com/yahoo/document/DocumentPut.java index e5ddc2c67a3..5906a9ca0ba 100644 --- a/document/src/main/java/com/yahoo/document/DocumentPut.java +++ b/document/src/main/java/com/yahoo/document/DocumentPut.java @@ -45,4 +45,9 @@ public class DocumentPut extends DocumentOperation { this.document = newDocument; } + @Override + public String toString() { + return "put of document " + getId(); + } + } diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index ec0d29a53a6..1b5681e7146 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -23,6 +23,7 @@ import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; * @author steinar */ public class JsonFeedReader implements FeedReader { + private final JsonReader reader; private InputStream stream; private static final JsonFactory jsonFactory = new JsonFactory().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES); diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index bedfbdc3da5..b7818b06b03 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -28,7 +28,6 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart * @author Steinar Knutsen * @author dybis */ -@Beta public class JsonReader { public Optional<DocumentParseInfo> parseDocument() throws IOException { @@ -79,6 +78,7 @@ public class JsonReader { return operation; } + /** Returns the next document operation, or null if we have reached the end */ public DocumentOperation next() { switch (state) { case AT_START: diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index e6fc8171a1a..e20845bfa54 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -13,9 +13,10 @@ import com.google.common.base.Preconditions; /** * Helper class to enable lookahead in the token stream. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class TokenBuffer { + public static final class Token { public final JsonToken token; public final String name; @@ -42,6 +43,9 @@ public class TokenBuffer { } } + /** Returns whether any tokens are available in this */ + public boolean isEmpty() { return size() == 0; } + public JsonToken next() { buffer.removeFirst(); Token t = buffer.peekFirst(); @@ -52,16 +56,25 @@ public class TokenBuffer { return t.token; } + /** Returns the current token without changing position, or null if none */ public JsonToken currentToken() { - return buffer.peekFirst().token; + Token token = buffer.peekFirst(); + if (token == null) return null; + return token.token; } + /** Returns the current token name without changing position, or null if none */ public String currentName() { - return buffer.peekFirst().name; + Token token = buffer.peekFirst(); + if (token == null) return null; + return token.name; } + /** Returns the current token text without changing position, or null if none */ public String currentText() { - return buffer.peekFirst().text; + Token token = buffer.peekFirst(); + if (token == null) return null; + return token.text; } public int size() { diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index 744ec12bb23..3fc2c941b99 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -33,31 +33,43 @@ public class DocumentParser { this.parser = parser; } + /** + * Parses a single document and returns it. + * Returns empty is we have reached the end of the stream. + */ public Optional<DocumentParseInfo> parse(Optional<DocumentId> documentIdArg) throws IOException { indentLevel = 0; DocumentParseInfo documentParseInfo = new DocumentParseInfo(); documentIdArg.ifPresent(documentId -> documentParseInfo.documentId = documentId); + boolean foundItems = false; do { - parseOneItem(documentParseInfo, documentIdArg.isPresent() /* doc id set externally */); + foundItems |= parseOneItem(documentParseInfo, documentIdArg.isPresent() /* doc id set externally */); } while (indentLevel > 0L); - if (documentParseInfo.documentId != null) { - return Optional.of(documentParseInfo); + if (documentParseInfo.documentId == null) { + if (foundItems) + throw new IllegalArgumentException("Missing a document operation ('put', 'update' or 'remove')"); + else + return Optional.empty(); } - return Optional.empty(); + return Optional.of(documentParseInfo); } - private void parseOneItem(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { + /** + * Parses one item from the stream. + * + * @return whether an item was found + */ + private boolean parseOneItem(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { parser.nextValue(); processIndent(); - if (parser.getCurrentName() == null) { - return; - } + if (parser.getCurrentName() == null) return false; if (indentLevel == 1L) { handleIdentLevelOne(documentParseInfo, docIdAndOperationIsSetExternally); } else if (indentLevel == 2L) { handleIdentLevelTwo(documentParseInfo); } + return true; } private void processIndent() { diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index 8e381c8e2fe..6189c12c8c9 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -74,6 +74,8 @@ public class VespaJsonDocumentReader { // Exposed for unit testing... public void readPut(TokenBuffer buffer, DocumentPut put) { try { + if (buffer.isEmpty()) // no "fields" map + throw new IllegalArgumentException(put + " is missing a 'fields' map"); populateComposite(buffer, put.getDocument()); } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, put.getId()); @@ -82,6 +84,8 @@ public class VespaJsonDocumentReader { // Exposed for unit testing... public void readUpdate(TokenBuffer buffer, DocumentUpdate update) { + if (buffer.isEmpty()) + throw new IllegalArgumentException("update of document " + update.getId() + " is missing a 'fields' map"); expectObjectStart(buffer.currentToken()); int localNesting = buffer.nesting(); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 07f0a172caf..1fd45cb07c4 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -1094,6 +1094,67 @@ public class JsonReaderTestCase { new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); } + @Test + public void testMissingOperation() { + try { + String jsonData = inputJson( + "[", + " {", + " 'fields': {", + " 'actualarray': {", + " 'add': [", + " 'person',", + " 'another person'", + " ]", + " }", + " }", + " }", + "]"); + + new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Missing a document operation ('put', 'update' or 'remove')", e.getMessage()); + } + } + + @Test + public void testMissingFieldsMapInPut() { + try { + String jsonData = inputJson( + "[", + " {", + " 'put': 'id:unittest:smoke::whee'", + " }", + "]"); + + new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("put of document id:unittest:smoke::whee is missing a 'fields' map", e.getMessage()); + } + } + + @Test + public void testMissingFieldsMapInUpdate() { + try { + String jsonData = inputJson( + "[", + " {", + " 'update': 'id:unittest:smoke::whee'", + " }", + "]"); + + new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("update of document id:unittest:smoke::whee is missing a 'fields' map", e.getMessage()); + } + } + static ByteArrayInputStream jsonToInputStream(String json) { return new ByteArrayInputStream(Utf8.toBytes(json)); } diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index b6aca1ec8be..97dd916bc60 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -200,8 +200,6 @@ DocumentUpdateTest::testSimpleUsage() { ByteBuffer::UP docBuf = serialize42(docUpdate); docBuf->flip(); DocumentUpdate::UP docUpdateCopy(DocumentUpdate::create42(repo, *docBuf)); - CPPUNIT_ASSERT(!docUpdate.affectsDocumentBody()); - CPPUNIT_ASSERT(!docUpdateCopy->affectsDocumentBody()); // Create a test document Document doc(*docType, DocumentId("doc::testdoc")); @@ -252,7 +250,6 @@ DocumentUpdateTest::testSimpleUsage() { updated.getValue("intarr").release())); CPPUNIT_ASSERT_EQUAL(size_t(3), val->size()); CPPUNIT_ASSERT_EQUAL(4, (*val)[2].getAsInt()); - CPPUNIT_ASSERT(upd.affectsDocumentBody()); } { Document updated(doc); @@ -593,24 +590,21 @@ void DocumentUpdateTest::testReadSerializedFile() DocumentUpdate& upd(*updp); const DocumentType *type = repo.getDocumentType("serializetest"); - CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")), - upd.getId()); + CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")), upd.getId()); CPPUNIT_ASSERT_EQUAL(*type, upd.getType()); // Verify assign value update. - FieldUpdate serField = upd[0]; + FieldUpdate serField = upd.getUpdates()[0]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("intfield").getId()); const ValueUpdate* serValue = &serField[0]; CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Assign); - const AssignValueUpdate* assign( - static_cast<const AssignValueUpdate*>(serValue)); - CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), - static_cast<const IntFieldValue&>(assign->getValue())); + const AssignValueUpdate* assign(static_cast<const AssignValueUpdate*>(serValue)); + CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), static_cast<const IntFieldValue&>(assign->getValue())); // Verify clear field update. - serField = upd[1]; + serField = upd.getUpdates()[1]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("floatfield").getId()); serValue = &serField[0]; @@ -618,7 +612,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId)); // Verify add value update. - serField = upd[2]; + serField = upd.getUpdates()[2]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("arrayoffloatfield").getId()); serValue = &serField[0]; @@ -765,14 +759,11 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("tags")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign array field with no array values = empty array. DocumentUpdate update(*doc->getDataType(), doc->getId()); - update.addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate( - ArrayFieldValue(field.getDataType())))); + update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(ArrayFieldValue(field.getDataType())))); update.applyTo(*doc); // Verify that the field was set in the document. @@ -781,10 +772,9 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove array field. - update.clear(); - update.addUpdate(FieldUpdate(field) - .addUpdate(ClearValueUpdate())); - update.applyTo(*doc); + DocumentUpdate update2(*doc->getDataType(), doc->getId()); + update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); + update2.applyTo(*doc); // Verify that the field was cleared in the document. std::unique_ptr<ArrayFieldValue> fval2(doc->getAs<ArrayFieldValue>(field)); @@ -798,31 +788,25 @@ DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("stringweightedset")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign weighted set with no items = empty set. DocumentUpdate update(*doc->getDataType(), doc->getId()); - update.addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate( - WeightedSetFieldValue(field.getDataType())))); + update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(WeightedSetFieldValue(field.getDataType())))); update.applyTo(*doc); // Verify that the field was set in the document. - std::unique_ptr<WeightedSetFieldValue> - fval1(doc->getAs<WeightedSetFieldValue>(field)); + auto fval1(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT(fval1.get()); CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove weighted set field. - update.clear(); - update.addUpdate(FieldUpdate(field) - .addUpdate(ClearValueUpdate())); - update.applyTo(*doc); + DocumentUpdate update2(*doc->getDataType(), doc->getId()); + update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); + update2.applyTo(*doc); // Verify that the field was cleared in the document. - std::unique_ptr<WeightedSetFieldValue> - fval2(doc->getAs<WeightedSetFieldValue>(field)); + auto fval2(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT(!fval2); } @@ -833,8 +817,7 @@ DocumentUpdateTest::testUpdateArrayWrongSubtype() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("tags")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign int values to string array. DocumentUpdate update(*doc->getDataType(), doc->getId()); diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 89f9c0dab67..7058af95828 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -221,12 +221,10 @@ ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - bool affectsBody = a.affectsDocumentBody(); ByteBuffer::UP bb(serializeHEAD(a)); bb->flip(); DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); - CPPUNIT_ASSERT_EQUAL(affectsBody, b->affectsDocumentBody()); CPPUNIT_ASSERT_EQUAL(size_t(0), bb->getRemaining()); CPPUNIT_ASSERT_EQUAL(a.getId().toString(), b->getId().toString()); CPPUNIT_ASSERT_EQUAL(a.getUpdates().size(), b->getUpdates().size()); @@ -1021,13 +1019,11 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() // structmap is body field { DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{janitor}", std::string(), fv4)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(update1); - CPPUNIT_ASSERT(docUp.affectsDocumentBody()); } // strfoo is header field @@ -1037,7 +1033,6 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() "strfoo", std::string(), StringFieldValue("helloworld"))); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(update1); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); } } @@ -1070,7 +1065,6 @@ FieldPathUpdateTestCase::testSerializeAssign() val.setValue("rating", IntFieldValue(100)); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{ribbit}", "true", val)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); @@ -1091,7 +1085,6 @@ FieldPathUpdateTestCase::testSerializeAdd() adds.add(StringFieldValue("george is getting upset!")); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AddFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), adds)); docUp.addFieldPathUpdate(update1); @@ -1106,7 +1099,6 @@ FieldPathUpdateTestCase::testSerializeRemove() MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new RemoveFieldPathUpdate("structmap{ribbit}", std::string())); docUp.addFieldPathUpdate(update1); diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 837ebd873e3..f9cf3472110 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -905,7 +905,7 @@ ArithmeticValueNode::getValue(std::unique_ptr<Value> lval, slval.getValue() + srval.getValue())); } } - //@fallthrough@ + [[fallthrough]]; case SUB: case MUL: case DIV: @@ -995,7 +995,7 @@ ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval, return result; } } - //@fallthrough@ + [[fallthrough]]; case SUB: case MUL: case DIV: diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index 3b0fe581c9e..818759d35b5 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -76,7 +76,7 @@ private: void write(const AssignFieldPathUpdate &value); void write(const RemoveFieldPathUpdate &value); - void visit(const DocumentUpdate &value) override { write42(value); } + void visit(const DocumentUpdate &value) override { writeHEAD(value); } void visit(const FieldUpdate &value) override { write(value); } void visit(const RemoveValueUpdate &value) override { write(value); } void visit(const AddValueUpdate &value) override { write(value); } diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index fa289ab0bfa..a3cc2a95155 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -21,27 +21,14 @@ using namespace vespalib::xml; namespace document { -IMPLEMENT_IDENTIFIABLE(DocumentUpdate, vespalib::Identifiable); - // Declare content bits. static const unsigned char CONTENT_HASTYPE = 0x01; -typedef std::vector<FieldUpdate> FieldUpdateList; -typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateList; - -DocumentUpdate::DocumentUpdate() - : _documentId("doc::"), - _type(DataType::DOCUMENT), - _updates(), - _version(Document::getNewestSerializationVersion()), - _createIfNonExistent(false) -{ -} - DocumentUpdate::DocumentUpdate(const DataType &type, const DocumentId& id) : _documentId(id), _type(&type), _updates(), + _fieldPathUpdates(), _version(Document::getNewestSerializationVersion()), _createIfNonExistent(false) { @@ -72,7 +59,7 @@ DocumentUpdate::DocumentUpdate(const DocumentTypeRepo& repo, } } -DocumentUpdate::~DocumentUpdate() { } +DocumentUpdate::~DocumentUpdate() = default; bool @@ -92,22 +79,6 @@ DocumentUpdate::operator==(const DocumentUpdate& other) const return true; } -bool -DocumentUpdate::affectsDocumentBody() const -{ - for(const auto & update : _updates) { - if (!update.getField().isHeaderField()) { - return true; - } - } - for (const auto & update : _fieldPathUpdates) { - if (update->affectsDocumentBody(*_type)) { - return true; - } - } - return false; -} - const DocumentType& DocumentUpdate::getType() const { return static_cast<const DocumentType &> (*_type); @@ -125,11 +96,6 @@ DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) { return *this; } -DocumentUpdate* -DocumentUpdate::clone() const { - return new DocumentUpdate(*this); -} - void DocumentUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -183,13 +149,6 @@ DocumentUpdate::applyTo(Document& doc) const } void -DocumentUpdate::serialize42(nbostream &stream) const -{ - VespaDocumentSerializer serializer(stream); - serializer.write42(*this); -} - -void DocumentUpdate::serializeHEAD(nbostream &stream) const { VespaDocumentSerializer serializer(stream); @@ -239,15 +198,13 @@ namespace { DocumentUpdate::UP DocumentUpdate::create42(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - return std::make_unique<DocumentUpdate>(repo, buffer, - SerializeVersion::SERIALIZE_42); + return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_42); } DocumentUpdate::UP DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - return std::make_unique<DocumentUpdate>(repo, buffer, - SerializeVersion::SERIALIZE_HEAD); + return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_HEAD); } void @@ -333,13 +290,6 @@ DocumentUpdate::deserializeFlags(int sizeAndFlags) } void -DocumentUpdate::onDeserialize42(const DocumentTypeRepo &repo, - ByteBuffer& buffer) -{ - deserialize42(repo, buffer); -} - -void DocumentUpdate::printXml(XmlOutputStream& xos) const { xos << XmlTag("document") diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index e6c39278ee2..a7f9d13d693 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -41,15 +41,9 @@ class Document; * path updates was added, and a new serialization format was * introduced while keeping the old one. */ -class DocumentUpdate : public vespalib::Identifiable, - public Printable, - public XmlSerializable +class DocumentUpdate final : public Printable, public XmlSerializable { -public: - typedef std::unique_ptr<DocumentUpdate> UP; - typedef std::shared_ptr<DocumentUpdate> SP; - typedef std::vector<FieldUpdate> FieldUpdateV; - typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV; +private: /** * Enum class containing the legal serialization version for * document updates. This version is not encoded in the serialized @@ -59,6 +53,11 @@ public: SERIALIZE_42, // old style format, before vespa 5.0 SERIALIZE_HEAD // new style format, since vespa 5.0 }; +public: + typedef std::unique_ptr<DocumentUpdate> UP; + typedef std::shared_ptr<DocumentUpdate> SP; + typedef std::vector<FieldUpdate> FieldUpdateV; + typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV; /** * Create old style document update, no support for field path updates. @@ -69,7 +68,16 @@ public: * Create new style document update, possibly with field path updates. */ static DocumentUpdate::UP createHEAD(const DocumentTypeRepo&, ByteBuffer&); - + + /** + * Create a document update from a byte buffer containing a serialized + * document update. Public to allow useage in std::make_unique/shared. + * + * @param repo Document type repo used to find proper document type + * @param buffer The buffer containing the serialized document update + * @param serializeVersion Selector between serialization formats. + */ + DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, SerializeVersion serializeVersion); /** * The document type is not strictly needed, as we know this at applyTo() * time, but search does not use applyTo() code to do the update, and don't @@ -82,18 +90,9 @@ public: */ DocumentUpdate(const DataType &type, const DocumentId& id); - /** - * Create a document update from a byte buffer containing a serialized - * document update. - * - * @param repo Document type repo used to find proper document type - * @param buffer The buffer containing the serialized document update - * @param serializeVersion Selector between serialization formats. - */ - DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, - SerializeVersion serializeVersion); - - ~DocumentUpdate(); + DocumentUpdate(const DocumentUpdate &) = delete; + DocumentUpdate & operator = (const DocumentUpdate &) = delete; + ~DocumentUpdate() override; bool operator==(const DocumentUpdate&) const; bool operator!=(const DocumentUpdate & rhs) const { return ! (*this == rhs); } @@ -107,11 +106,6 @@ public: * type as this. */ void applyTo(Document& doc) const; - - void clear() { _updates.clear(); } - size_t size() const { return _updates.size(); } - const FieldUpdate& operator[](int index) const { return _updates[index]; } - FieldUpdate& operator[](int index) { return _updates[index]; } /** * Add a field update to this document update. @@ -131,26 +125,15 @@ public: /** @return The list of fieldpath updates. */ const FieldPathUpdateV & getFieldPathUpdates() const { return _fieldPathUpdates; } - bool affectsDocumentBody() const; - /** @return The type of document this update is for. */ const DocumentType& getType() const; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize42(const DocumentTypeRepo&, ByteBuffer&); - void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&); - - // Deserializable implementation. Kept as search relies on it currently. - virtual void onDeserialize42(const DocumentTypeRepo &repo, ByteBuffer&); - - void serialize42(vespalib::nbostream &stream) const; void serializeHEAD(vespalib::nbostream &stream) const; void printXml(XmlOutputStream&) const override; - virtual DocumentUpdate* clone() const; - /** * Sets whether this update should create the document it updates if that document does not exist. * In this case an empty document is created before the update is applied. @@ -169,7 +152,6 @@ public: int serializeFlags(int size_) const; int16_t getVersion() const { return _version; } - DECLARE_IDENTIFIABLE(DocumentUpdate); private: DocumentId _documentId; // The ID of the document to update. const DataType *_type; // The type of document this update is for. @@ -178,15 +160,9 @@ private: int16_t _version; // Serialization version bool _createIfNonExistent; - /** - * This function exist because search relies on deserialization through - * creating object through empty constructor and calling deserialize. - * - * It is hidden to prevent accidental other usage. - */ - DocumentUpdate(); - int deserializeFlags(int sizeAndFlags); + void deserialize42(const DocumentTypeRepo&, ByteBuffer&); + void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&); }; } // document diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 7265329d4d5..239ffff2fef 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -45,7 +45,7 @@ FieldPathUpdate::FieldPathUpdate(stringref fieldPath, stringref whereClause) : _originalWhereClause(whereClause) { } -FieldPathUpdate::~FieldPathUpdate() { } +FieldPathUpdate::~FieldPathUpdate() = default; bool FieldPathUpdate::operator==(const FieldPathUpdate& other) const @@ -76,16 +76,6 @@ FieldPathUpdate::applyTo(Document& doc) const } } -bool -FieldPathUpdate::affectsDocumentBody(const DataType & type) const -{ - FieldPath path; - type.buildFieldPath(path, _originalFieldPath); - if (path.empty() || !path[0].hasField()) return false; - const Field& field = path[0].getFieldRef(); - return !field.isHeaderField(); -} - void FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const { diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index c120c8d3979..6629aa74aee 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -64,9 +64,6 @@ public: */ void checkCompatibility(const FieldValue& fv, const DataType & type) const; - /** @return Whether or not the first field path element is a body field */ - bool affectsDocumentBody(const DataType & type) const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_IDENTIFIABLE_ABSTRACT(FieldPathUpdate); diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index 57396da15ac..277d6467ae8 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -16,21 +16,32 @@ FieldUpdate::FieldUpdate(const Field& field) { } -// Construct a field update by deserialization. -FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, - const DocumentType& type, - ByteBuffer& buffer, - int16_t version) +namespace { + +int readInt(ByteBuffer & buffer) { + int tmp; + buffer.getIntNetwork(tmp); + return tmp; +} + +} + +FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type, ByteBuffer& buffer, int16_t version) : Printable(), - _field(), + _field(type.getField(readInt(buffer))), _updates() { - deserialize(repo, type, buffer, version); + int numUpdates = readInt(buffer); + _updates.reserve(numUpdates); + const DataType& dataType = _field.getDataType(); + for(int i(0); i < numUpdates; i++) { + _updates.emplace_back(ValueUpdate::createInstance(repo, dataType, buffer, version).release()); + } } FieldUpdate::FieldUpdate(const FieldUpdate &) = default; FieldUpdate & FieldUpdate::operator = (const FieldUpdate &) = default; -FieldUpdate::~FieldUpdate() {} +FieldUpdate::~FieldUpdate() = default; bool FieldUpdate::operator==(const FieldUpdate& other) const @@ -77,8 +88,7 @@ FieldUpdate::applyTo(Document& doc) const // Print this field update as a human readable string. void -FieldUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +FieldUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "FieldUpdate(" << _field.toString(verbose); for(const auto & update : _updates) { @@ -93,8 +103,8 @@ FieldUpdate::print(std::ostream& out, bool verbose, // Deserialize this field update from the given buffer. void -FieldUpdate::deserialize(const DocumentTypeRepo& repo, - const DocumentType& docType, ByteBuffer& buffer, int16_t version) +FieldUpdate::deserialize(const DocumentTypeRepo& repo, const DocumentType& docType, + ByteBuffer& buffer, int16_t version) { int fieldId; buffer.getIntNetwork(fieldId); diff --git a/document/src/vespa/document/update/fieldupdate.h b/document/src/vespa/document/update/fieldupdate.h index 3f7b3dde3f7..569fb21fe1c 100644 --- a/document/src/vespa/document/update/fieldupdate.h +++ b/document/src/vespa/document/update/fieldupdate.h @@ -49,7 +49,7 @@ public: * @param serializationVersion The serialization version the update was serialized with. */ FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type, - ByteBuffer& buffer, int16_t serializationVersion); + ByteBuffer& buffer, int16_t version); bool operator==(const FieldUpdate&) const; bool operator!=(const FieldUpdate & rhs) const { return ! (*this == rhs); } diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 0346cadbf9b..48728ff6057 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -7,6 +7,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> using document::DataType; using document::DocumentTypeRepo; @@ -240,6 +241,8 @@ Messages50Test::testRemoveLocationMessage() if (EXPECT_TRUE(obj.get() != NULL)) { RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj); EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection()); + // FIXME add to wire format, currently hardcoded. + EXPECT_EQUAL(string(document::FixedBucketSpaces::default_space_name()), ref.getBucketSpace()); } } } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index b5a4a8306b1..26d85b57522 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -32,7 +32,7 @@ RoutableFactories50::DocumentMessageFactory::decode(document::ByteBuffer &in, co uint32_t loadClass = decodeInt(in); DocumentMessage::UP msg = doDecode(in); - if (msg.get() != NULL) { + if (msg) { msg->setPriority((Priority::Value)pri); msg->setLoadType(loadTypes[loadClass]); } @@ -54,7 +54,7 @@ RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, cons uint8_t pri; in.getByte(pri); DocumentReply::UP reply = doDecode(in); - if (reply.get() != NULL) { + if (reply) { reply->setPriority((Priority::Value)pri); } return mbus::Routable::UP(reply.release()); @@ -72,22 +72,17 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteB uint64_t userId = (uint64_t)decodeLong(buf); string group = decodeString(buf); - BatchDocumentUpdateMessage* msg; - if (group.length()) { - msg = new BatchDocumentUpdateMessage(group); - } else { - msg = new BatchDocumentUpdateMessage(userId); - } - DocumentMessage::UP retVal(msg); + auto msg = (group.length()) + ? std::make_unique<BatchDocumentUpdateMessage>(group) + : std::make_unique<BatchDocumentUpdateMessage>(userId); uint32_t len = decodeInt(buf); for (uint32_t i = 0; i < len; i++) { - document::DocumentUpdate::SP upd; - upd.reset(document::DocumentUpdate::createHEAD(_repo, buf).release()); + document::DocumentUpdate::SP upd = document::DocumentUpdate::createHEAD(_repo, buf); msg->addUpdate(upd); } - return retVal; + return msg; } bool @@ -111,14 +106,14 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentM DocumentReply::UP RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const { - BatchDocumentUpdateReply* reply = new BatchDocumentUpdateReply(); + auto reply = std::make_unique<BatchDocumentUpdateReply>(); reply->setHighestModificationTimestamp(decodeLong(buf)); std::vector<bool>& notFound = reply->getDocumentsNotFound(); notFound.resize(decodeInt(buf)); for (std::size_t i = 0; i < notFound.size(); ++i) { notFound[i] = decodeBoolean(buf); } - return DocumentReply::UP(reply); + return reply; } bool @@ -126,10 +121,10 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep { const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj); buf.putLong(reply.getHighestModificationTimestamp()); - const std::vector<bool>& notFound = reply.getDocumentsNotFound(); - buf.putInt(notFound.size()); - for (std::size_t i = 0; i < notFound.size(); ++i) { - buf.putBoolean(notFound[i]); + const std::vector<bool>& notFoundV = reply.getDocumentsNotFound(); + buf.putInt(notFoundV.size()); + for (bool notFound : notFoundV) { + buf.putBoolean(notFound); } return true; } @@ -137,34 +132,33 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep DocumentMessage::UP RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new CreateVisitorMessage()); - CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret); + auto msg = std::make_unique<CreateVisitorMessage>(); - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaximumPendingReplyCount(decodeInt(buf)); + msg->setLibraryName(decodeString(buf)); + msg->setInstanceId(decodeString(buf)); + msg->setControlDestination(decodeString(buf)); + msg->setDataDestination(decodeString(buf)); + msg->setDocumentSelection(decodeString(buf)); + msg->setMaximumPendingReplyCount(decodeInt(buf)); int32_t len = decodeInt(buf); for (int32_t i = 0; i < len; i++) { int64_t val; buf.getLong(val); // NOT using getLongNetwork - msg.getBuckets().push_back(document::BucketId(val)); + msg->getBuckets().push_back(document::BucketId(val)); } - msg.setFromTimestamp(decodeLong(buf)); - msg.setToTimestamp(decodeLong(buf)); - msg.setVisitRemoves(decodeBoolean(buf)); - msg.setVisitHeadersOnly(decodeBoolean(buf)); - msg.setVisitInconsistentBuckets(decodeBoolean(buf)); - msg.getParameters().deserialize(_repo, buf); - msg.setVisitorDispatcherVersion(50); - msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); - msg.setMaxBucketsPerVisitor(decodeInt(buf)); + msg->setFromTimestamp(decodeLong(buf)); + msg->setToTimestamp(decodeLong(buf)); + msg->setVisitRemoves(decodeBoolean(buf)); + msg->setVisitHeadersOnly(decodeBoolean(buf)); + msg->setVisitInconsistentBuckets(decodeBoolean(buf)); + msg->getParameters().deserialize(_repo, buf); + msg->setVisitorDispatcherVersion(50); + msg->setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); + msg->setMaxBucketsPerVisitor(decodeInt(buf)); - return ret; + return msg; } bool @@ -180,11 +174,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage buf.putInt(msg.getMaximumPendingReplyCount()); buf.putInt(msg.getBuckets().size()); - const std::vector<document::BucketId> &buckets = msg.getBuckets(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - uint64_t val = it->getRawId(); + for (const auto & bucketId : msg.getBuckets()) { + uint64_t val = bucketId.getRawId(); buf.putBytes((const char*)&val, 8); } @@ -208,9 +199,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage DocumentReply::UP RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); - CreateVisitorReply &reply = static_cast<CreateVisitorReply&>(*ret); - reply.setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); + auto reply = std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); + reply->setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); vdslib::VisitorStatistics vs; vs.setBucketsVisited(decodeInt(buf)); vs.setDocumentsVisited(decodeLong(buf)); @@ -219,9 +209,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b vs.setBytesReturned(decodeLong(buf)); vs.setSecondPassDocumentsReturned(decodeLong(buf)); vs.setSecondPassBytesReturned(decodeLong(buf)); - reply.setVisitorStatistics(vs); + reply->setVisitorStatistics(vs); - return ret; + return reply; } bool @@ -242,10 +232,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doEncode(const DocumentReply &ob DocumentMessage::UP RoutableFactories50::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DestroyVisitorMessage()); - DestroyVisitorMessage &msg = static_cast<DestroyVisitorMessage&>(*ret); - msg.setInstanceId(decodeString(buf)); - return ret; + auto msg = std::make_unique<DestroyVisitorMessage>(); + msg->setInstanceId(decodeString(buf)); + return msg; } bool @@ -257,35 +246,30 @@ RoutableFactories50::DestroyVisitorMessageFactory::doEncode(const DocumentMessag } DocumentReply::UP -RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DESTROYVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DESTROYVISITOR); } bool -RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DocumentListMessage()); - DocumentListMessage &msg = static_cast<DocumentListMessage&>(*ret); - - msg.setBucketId(document::BucketId(decodeLong(buf))); + auto msg = std::make_unique<DocumentListMessage>(); + msg->setBucketId(document::BucketId(decodeLong(buf))); int32_t len = decodeInt(buf); for (int32_t i = 0; i < len; i++) { DocumentListMessage::Entry entry(_repo, buf); - msg.getDocuments().push_back(entry); + msg->getDocuments().push_back(entry); } - return ret; + return msg; } bool @@ -295,40 +279,36 @@ RoutableFactories50::DocumentListMessageFactory::doEncode(const DocumentMessage buf.putLong(msg.getBucketId().getRawId()); buf.putInt(msg.getDocuments().size()); - for (uint32_t i = 0; i < msg.getDocuments().size(); i++) { - int len = msg.getDocuments()[i].getSerializedSize(); + for (const auto & document : msg.getDocuments()) { + int len = document.getSerializedSize(); char *tmp = buf.allocate(len); document::ByteBuffer dbuf(tmp, len); - msg.getDocuments()[i].serialize(dbuf); + document.serialize(dbuf); } return true; } DocumentReply::UP -RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); } bool -RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DocumentSummaryMessage()); - DocumentSummaryMessage &msg = static_cast<DocumentSummaryMessage&>(*ret); + auto msg = std::make_unique<DocumentSummaryMessage>(); - msg.deserialize(buf); + msg->deserialize(buf); - return ret; + return msg; } bool @@ -345,34 +325,30 @@ RoutableFactories50::DocumentSummaryMessageFactory::doEncode(const DocumentMessa } DocumentReply::UP -RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); } bool -RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new EmptyBucketsMessage()); - EmptyBucketsMessage &msg = static_cast<EmptyBucketsMessage&>(*ret); + auto msg = std::make_unique<EmptyBucketsMessage>(); int32_t len = decodeInt(buf); std::vector<document::BucketId> buckets(len); for (int32_t i = 0; i < len; ++i) { buckets[i] = document::BucketId(decodeLong(buf)); } - msg.getBucketIds().swap(buckets); + msg->getBucketIds().swap(buckets); - return ret; + return msg; } bool @@ -381,35 +357,28 @@ RoutableFactories50::EmptyBucketsMessageFactory::doEncode(const DocumentMessage const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); buf.putInt(msg.getBucketIds().size()); - const std::vector<document::BucketId> &buckets = msg.getBucketIds(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - buf.putLong(it->getRawId()); + for (const auto & bucketId : msg.getBucketIds()) { + buf.putLong(bucketId.getRawId()); } return true; } DocumentReply::UP -RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_EMPTYBUCKETS)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_EMPTYBUCKETS); } bool -RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } -bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; +bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer& ) const +{ return (bucketSpace == FixedBucketSpaces::default_space_name()); } @@ -437,18 +406,18 @@ RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage DocumentReply::UP RoutableFactories50::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetBucketListReply()); - GetBucketListReply &reply = static_cast<GetBucketListReply&>(*ret); + auto reply = std::make_unique<GetBucketListReply>(); int32_t len = decodeInt(buf); + reply->getBuckets().reserve(len); for (int32_t i = 0; i < len; i++) { GetBucketListReply::BucketInfo info; info._bucket = document::BucketId((uint64_t)decodeLong(buf)); info._bucketInformation = decodeString(buf); - reply.getBuckets().push_back(info); + reply->getBuckets().push_back(info); } - return ret; + return reply; } bool @@ -458,11 +427,9 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); buf.putInt(buckets.size()); - for (std::vector<GetBucketListReply::BucketInfo>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - buf.putLong(it->_bucket.getRawId()); - buf.putString(it->_bucketInformation); + for (const auto & bucketInfo : buckets) { + buf.putLong(bucketInfo._bucket.getRawId()); + buf.putString(bucketInfo._bucketInformation); } return true; @@ -471,12 +438,11 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob DocumentMessage::UP RoutableFactories50::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new GetBucketStateMessage()); - GetBucketStateMessage &msg = static_cast<GetBucketStateMessage&>(*ret); + auto msg = std::make_unique<GetBucketStateMessage>(); - msg.setBucketId(document::BucketId((uint64_t)decodeLong(buf))); + msg->setBucketId(document::BucketId((uint64_t)decodeLong(buf))); - return ret; + return msg; } bool @@ -490,16 +456,15 @@ RoutableFactories50::GetBucketStateMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetBucketStateReply()); - GetBucketStateReply &reply = static_cast<GetBucketStateReply&>(*ret); + auto reply = std::make_unique<GetBucketStateReply>(); int32_t len = decodeInt(buf); + reply->getBucketState().reserve(len); for (int32_t i = 0; i < len; i++) { - DocumentState state(buf); - reply.getBucketState().push_back(state); + reply->getBucketState().emplace_back(buf); } - return ret; + return reply; } bool @@ -508,11 +473,8 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj); buf.putInt(reply.getBucketState().size()); - const std::vector<DocumentState> &state = reply.getBucketState(); - for (std::vector<DocumentState>::const_iterator it = state.begin(); - it != state.end(); ++it) - { - it->serialize(buf); + for (const auto & state : reply.getBucketState()) { + state.serialize(buf); } return true; @@ -521,13 +483,11 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories50::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new GetDocumentMessage()); - GetDocumentMessage &msg = static_cast<GetDocumentMessage&>(*ret); - - msg.setDocumentId(decodeDocumentId(buf)); - msg.setFlags(decodeInt(buf)); + auto msg = std::make_unique<GetDocumentMessage>(); + msg->setDocumentId(decodeDocumentId(buf)); + msg->setFlags(decodeInt(buf)); - return ret; + return msg; } bool @@ -544,23 +504,22 @@ RoutableFactories50::GetDocumentMessageFactory::doEncode(const DocumentMessage & DocumentReply::UP RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetDocumentReply()); - GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret); + auto reply = std::make_unique<GetDocumentReply>(); bool hasDocument = decodeBoolean(buf); document::Document * document = nullptr; if (hasDocument) { auto doc = std::make_shared<document::Document>(_repo, buf); document = doc.get(); - reply.setDocument(std::move(doc)); + reply->setDocument(std::move(doc)); } int64_t lastModified = decodeLong(buf); - reply.setLastModified(lastModified); + reply->setLastModified(lastModified); if (hasDocument) { document->setLastModified(lastModified); } - return ret; + return reply; } bool @@ -582,12 +541,9 @@ RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, DocumentMessage::UP RoutableFactories50::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new MapVisitorMessage()); - MapVisitorMessage &msg = static_cast<MapVisitorMessage&>(*ret); - - msg.getData().deserialize(_repo, buf); - - return ret; + auto msg = std::make_unique<MapVisitorMessage>(); + msg->getData().deserialize(_repo, buf); + return msg; } bool @@ -604,17 +560,14 @@ RoutableFactories50::MapVisitorMessageFactory::doEncode(const DocumentMessage &o } DocumentReply::UP -RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); } bool -RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } @@ -672,11 +625,10 @@ RoutableFactories50::RemoveDocumentMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new RemoveDocumentReply()); - RemoveDocumentReply &reply = static_cast<RemoveDocumentReply&>(*ret); - reply.setWasFound(decodeBoolean(buf)); - reply.setHighestModificationTimestamp(decodeLong(buf)); - return ret; + auto reply = std::make_unique<RemoveDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; } bool @@ -696,7 +648,10 @@ RoutableFactories50::RemoveLocationMessageFactory::doDecode(document::ByteBuffer document::BucketIdFactory factory; document::select::Parser parser(_repo, factory); - return DocumentMessage::UP(new RemoveLocationMessage(factory, parser, selection)); + auto msg = std::make_unique<RemoveLocationMessage>(factory, parser, selection); + // FIXME bucket space not part of wire format, implicitly limiting to only default space for now. + msg->setBucketSpace(document::FixedBucketSpaces::default_space_name()); + return msg; } bool @@ -710,7 +665,7 @@ RoutableFactories50::RemoveLocationMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const { - return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_REMOVELOCATION)); + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_REMOVELOCATION); } bool @@ -722,12 +677,9 @@ RoutableFactories50::RemoveLocationReplyFactory::doEncode(const DocumentReply &, DocumentMessage::UP RoutableFactories50::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new SearchResultMessage()); - SearchResultMessage &msg = static_cast<SearchResultMessage&>(*ret); - - msg.deserialize(buf); - - return ret; + auto msg = std::make_unique<SearchResultMessage>(); + msg->deserialize(buf); + return msg; } bool @@ -746,13 +698,11 @@ RoutableFactories50::SearchResultMessageFactory::doEncode(const DocumentMessage DocumentMessage::UP RoutableFactories50::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new QueryResultMessage()); - QueryResultMessage &msg = static_cast<QueryResultMessage&>(*ret); - - msg.getSearchResult().deserialize(buf); - msg.getDocumentSummary().deserialize(buf); + auto msg = std::make_unique<QueryResultMessage>(); + msg->getSearchResult().deserialize(buf); + msg->getDocumentSummary().deserialize(buf); - return ret; + return msg; } bool @@ -770,39 +720,32 @@ RoutableFactories50::QueryResultMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); } bool -RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentReply::UP -RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); } bool -RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } -bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; +bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer& ) const +{ return (bucketSpace == FixedBucketSpaces::default_space_name()); } @@ -813,14 +756,13 @@ string RoutableFactories50::StatBucketMessageFactory::decodeBucketSpace(document DocumentMessage::UP RoutableFactories50::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new StatBucketMessage()); - StatBucketMessage &msg = static_cast<StatBucketMessage&>(*ret); + auto msg = std::make_unique<StatBucketMessage>(); - msg.setBucketId(document::BucketId(decodeLong(buf))); - msg.setDocumentSelection(decodeString(buf)); - msg.setBucketSpace(decodeBucketSpace(buf)); + msg->setBucketId(document::BucketId(decodeLong(buf))); + msg->setDocumentSelection(decodeString(buf)); + msg->setBucketSpace(decodeBucketSpace(buf)); - return ret; + return msg; } bool @@ -836,12 +778,9 @@ RoutableFactories50::StatBucketMessageFactory::doEncode(const DocumentMessage &o DocumentReply::UP RoutableFactories50::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new StatBucketReply()); - StatBucketReply &reply = static_cast<StatBucketReply&>(*ret); - - reply.setResults(decodeString(buf)); - - return ret; + auto reply = std::make_unique<StatBucketReply>(); + reply->setResults(decodeString(buf)); + return reply; } bool @@ -853,41 +792,32 @@ RoutableFactories50::StatBucketReplyFactory::doEncode(const DocumentReply &obj, } DocumentMessage::UP -RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &) const { - (void)buf; return DocumentMessage::UP(); // TODO: remove message type } bool -RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return false; } DocumentReply::UP -RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; return DocumentReply::UP(); // TODO: remove reply type } bool -RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return false; } void RoutableFactories50::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(make_shared<document::DocumentUpdate> - (_repo, buf, - document::DocumentUpdate::SerializeVersion:: - SERIALIZE_HEAD)); + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); } @@ -909,11 +839,10 @@ RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new UpdateDocumentReply()); - UpdateDocumentReply &reply = static_cast<UpdateDocumentReply&>(*ret); - reply.setWasFound(decodeBoolean(buf)); - reply.setHighestModificationTimestamp(decodeLong(buf)); - return ret; + auto reply = std::make_unique<UpdateDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; } bool @@ -928,18 +857,18 @@ RoutableFactories50::UpdateDocumentReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories50::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new VisitorInfoMessage()); - VisitorInfoMessage &msg = static_cast<VisitorInfoMessage&>(*ret); + auto msg = std::make_unique<VisitorInfoMessage>(); int32_t len = decodeInt(buf); + msg->getFinishedBuckets().reserve(len); for (int32_t i = 0; i < len; i++) { int64_t val; buf.getLong(val); // NOT using getLongNetwork - msg.getFinishedBuckets().push_back(document::BucketId(val)); + msg->getFinishedBuckets().emplace_back(val); } - msg.setErrorMessage(decodeString(buf)); + msg->setErrorMessage(decodeString(buf)); - return ret; + return msg; } bool @@ -948,11 +877,8 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage & const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); buf.putInt(msg.getFinishedBuckets().size()); - const std::vector<document::BucketId> &buckets = msg.getFinishedBuckets(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - uint64_t val = it->getRawId(); + for (const auto & bucketId : msg.getFinishedBuckets()) { + uint64_t val = bucketId.getRawId(); buf.putBytes((const char*)&val, 8); } buf.putString(msg.getErrorMessage()); @@ -961,29 +887,23 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); } bool -RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentReply::UP RoutableFactories50::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new WrongDistributionReply()); - WrongDistributionReply &reply = static_cast<WrongDistributionReply&>(*ret); - - reply.setSystemState(decodeString(buf)); - - return ret; + auto reply = std::make_unique<WrongDistributionReply>(); + reply->setSystemState(decodeString(buf)); + return reply; } bool @@ -1013,19 +933,19 @@ RoutableFactories50::FeedMessageFactory::myEncode(const FeedMessage &msg, vespal DocumentReply::UP RoutableFactories50::FeedReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new FeedReply(getType())); - FeedReply &reply = static_cast<FeedReply&>(*ret); + auto reply = std::make_unique<FeedReply>(getType()); - std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); + std::vector<FeedAnswer> &answers = reply->getFeedAnswers(); int32_t len = decodeInt(buf); + answers.reserve(len); for (int32_t i = 0; i < len; ++i) { int32_t typeCode = decodeInt(buf); int32_t wantedIncrement = decodeInt(buf); string recipient = decodeString(buf); string moreInfo = decodeString(buf); - answers.push_back(FeedAnswer(typeCode, wantedIncrement, recipient, moreInfo)); + answers.emplace_back(typeCode, wantedIncrement, recipient, moreInfo); } - return ret; + return reply; } bool @@ -1033,14 +953,11 @@ RoutableFactories50::FeedReplyFactory::doEncode(const DocumentReply &obj, vespal { const FeedReply &reply = static_cast<const FeedReply&>(obj); buf.putInt(reply.getFeedAnswers().size()); - const std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); - for (std::vector<FeedAnswer>::const_iterator it = answers.begin(); - it != answers.end(); ++it) - { - buf.putInt(it->getAnswerCode()); - buf.putInt(it->getWantedIncrement()); - buf.putString(it->getRecipient()); - buf.putString(it->getMoreInfo()); + for (const auto & answer : reply.getFeedAnswers()) { + buf.putInt(answer.getAnswerCode()); + buf.putInt(answer.getWantedIncrement()); + buf.putString(answer.getRecipient()); + buf.putString(answer.getMoreInfo()); } return true; } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp index ce83a2cd638..c7f3401d3e1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -17,8 +17,7 @@ RoutableRepository::VersionMap::VersionMap() : { } bool -RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, - IRoutableFactory::SP factory) +RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, IRoutableFactory::SP factory) { bool ret = _factoryVersions.find(version) != _factoryVersions.end(); _factoryVersions[version] = factory; @@ -41,7 +40,8 @@ RoutableRepository::VersionMap::getFactory(const vespalib::Version &version) con return IRoutableFactory::SP(); } - return std::max_element(candidates.begin(), candidates.end(), [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second; + return std::max_element(candidates.begin(), candidates.end(), + [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second; } RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) : @@ -50,7 +50,6 @@ RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) : _cache(), _loadTypes(loadTypes) { - // empty } mbus::Routable::UP @@ -65,13 +64,13 @@ RoutableRepository::decode(const vespalib::Version &version, mbus::BlobRef data) int type; in.getIntNetwork(type); IRoutableFactory::SP factory = getFactory(version, type); - if (factory.get() == NULL) { + if (!factory) { LOG(error, "No routable factory found for routable type %d (version %s).", type, version.toString().c_str()); return mbus::Routable::UP(); } mbus::Routable::UP ret = factory->decode(in, _loadTypes); - if (ret.get() == NULL) { + if (!ret) { LOG(error, "Routable factory failed to deserialize routable of type %d (version %s).", type, version.toString().c_str()); @@ -89,7 +88,7 @@ RoutableRepository::encode(const vespalib::Version &version, const mbus::Routabl uint32_t type = obj.getType(); IRoutableFactory::SP factory = getFactory(version, type); - if (factory.get() == NULL) { + if (!factory) { LOG(error, "No routable factory found for routable type %d (version %s).", type, version.toString().c_str()); return mbus::Blob(0); @@ -130,7 +129,7 @@ RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type) return IRoutableFactory::SP(); } IRoutableFactory::SP factory = vit->second.getFactory(version); - if (factory.get() == NULL) { + if (!factory) { return IRoutableFactory::SP(); } _cache[cacheKey] = factory; @@ -141,11 +140,9 @@ uint32_t RoutableRepository::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const { vespalib::LockGuard guard(_lock); - for (TypeMap::const_iterator it = _factoryTypes.begin(); - it != _factoryTypes.end(); ++it) - { - if (it->second.getFactory(version).get() != NULL) { - out.push_back(it->first); + for (const auto & type : _factoryTypes) { + if (type.second.getFactory(version)) { + out.push_back(type.first); } } return _factoryTypes.size(); diff --git a/fastlib/src/vespa/fastlib/util/base64.cpp b/fastlib/src/vespa/fastlib/util/base64.cpp index 1b7889d8c47..8b9ac45e698 100644 --- a/fastlib/src/vespa/fastlib/util/base64.cpp +++ b/fastlib/src/vespa/fastlib/util/base64.cpp @@ -90,7 +90,7 @@ Fast_Base64::Decode(const char *source, unsigned int length, char *destination) if (symbol != '=' || i == length) return -1; symbol = source[++i]; - //@fallthrough@ + [[fallthrough]]; case 3: for (; i < length; ++i) { symbol = source[i]; if (symbol == '\0') diff --git a/fat-model-dependencies/OWNERS b/fat-model-dependencies/OWNERS new file mode 100644 index 00000000000..d34761f1ba5 --- /dev/null +++ b/fat-model-dependencies/OWNERS @@ -0,0 +1,2 @@ +gjoranv +hmusum diff --git a/fat-model-dependencies/README b/fat-model-dependencies/README new file mode 100644 index 00000000000..ba71b189db9 --- /dev/null +++ b/fat-model-dependencies/README @@ -0,0 +1,4 @@ +This module contains all dependencies that must be embedded in the config-model-fat bundle. +This artifact should be depended on by config-model-fat and all amended versions of the +fat config model. This allows pulling in the same set of dependencies without duplication +in pom.xml. diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml new file mode 100644 index 00000000000..1415ca6e5aa --- /dev/null +++ b/fat-model-dependencies/pom.xml @@ -0,0 +1,223 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>fat-model-dependencies</artifactId> + <packaging>pom</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Large, and installed separately as part of Vespa --> + <groupId>org.tensorflow</groupId> + <artifactId>libtensorflow_jni</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>provided-dependencies</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-application-package</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configgen</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>simplemetrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>documentapi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vdslib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>statistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>processing</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>chain</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search-and-docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>logd</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchcore</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>storage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vsm</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>indexinglanguage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchsummary</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.major-version}</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> diff --git a/fbench/src/filterfile/filterfile.cpp b/fbench/src/filterfile/filterfile.cpp index ca93b70a046..e9b35de97e0 100644 --- a/fbench/src/filterfile/filterfile.cpp +++ b/fbench/src/filterfile/filterfile.cpp @@ -133,7 +133,7 @@ main(int argc, char** argv) break; case 1: buf[outIdx++] = line[idx]; - //@fallthrough@ + [[fallthrough]]; case 2: if (line[idx++] == '&') state = 0; diff --git a/fsa/src/vespa/fsa/segmenter.cpp b/fsa/src/vespa/fsa/segmenter.cpp index d13249a5ce8..3bcb3f1b489 100644 --- a/fsa/src/vespa/fsa/segmenter.cpp +++ b/fsa/src/vespa/fsa/segmenter.cpp @@ -77,16 +77,16 @@ void Segmenter::Segments::buildSegmentation(Segmenter::SegmentationMethod method switch(method){ case SEGMENTATION_WEIGHTED_BIAS100: bias+=50; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS50: bias+=30; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS20: bias+=10; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED_BIAS10: bias+=10; - //@fallthrough@ + [[fallthrough]]; case SEGMENTATION_WEIGHTED: bestid=-1; for(i=n_txt;i>=0;i--){ diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerActivator.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerActivator.java index 9d1b613e23c..59db453e2c4 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerActivator.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerActivator.java @@ -13,27 +13,28 @@ import com.yahoo.jdisc.Container; * #newContainerBuilder()}, 2) configure the returned {@link ContainerBuilder}, and 3) pass the builder to the {@link * #activateContainer(ContainerBuilder)} method.</p> * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public interface ContainerActivator { /** - * <p>This method creates and returns a new {@link ContainerBuilder} object that has the necessary references to the - * application and its internal components.</p> + * This method creates and returns a new {@link ContainerBuilder} object that has the necessary references to the + * application and its internal components. * * @return The created builder. */ - public ContainerBuilder newContainerBuilder(); + ContainerBuilder newContainerBuilder(); /** - * <p>Creates and activates a {@link Container} based on the provided {@link ContainerBuilder}. By providing a + * Creates and activates a {@link Container} based on the provided {@link ContainerBuilder}. By providing a * <em>null</em> argument, this method can be used to deactivate the current Container. The returned object can be - * used to schedule a cleanup task that is executed once the the deactivated Container has terminated.</p> + * used to schedule a cleanup task that is executed once the the deactivated Container has terminated. * * @param builder The builder to activate. * @return The previous container, if any. * @throws ApplicationNotReadyException If this method is called before {@link Application#start()} or after {@link * Application#stop()}. */ - public DeactivatedContainer activateContainer(ContainerBuilder builder); + DeactivatedContainer activateContainer(ContainerBuilder builder); + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index d491daf55dd..c85917c4c7e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -360,10 +360,11 @@ public class JettyHttpServer extends AbstractServerProvider { return statisticsHandler; } + @SuppressWarnings("deprecation") private GzipHandler newGzipHandler(ServerConfig serverConfig) { GzipHandler gzipHandler = new GzipHandlerWithVaryHeaderFixed(); gzipHandler.setCompressionLevel(serverConfig.responseCompressionLevel()); - gzipHandler.setCheckGzExists(false); + gzipHandler.setCheckGzExists(false); // TODO: will be removed without replacement in Jetty 10 gzipHandler.setIncludedMethods("GET", "POST"); return gzipHandler; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java index d2c09aae22a..71e55c36284 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java @@ -2,31 +2,33 @@ package com.yahoo.vespa.hosted.node.admin.component; /** - * This class is thread unsafe: All method calls MUST be exclusive and serialized. + * <p>This class is thread unsafe: All method calls MUST be exclusive and serialized.</p> * - * In a specialized environment it is possible to provide a richer context than TaskContext: - * - Define a subclass T of TaskContext with the additional functionality. - * - Define task classes that implement IdempotentTask<T>. + * <dl> + * <dt>In a specialized environment it is possible to provide a richer context than TaskContext:</dt> + * <dd>- Define a subclass T of TaskContext with the additional functionality.</dd> + * <dd>- Define task classes that implement IdempotentTask<T>.</dd> + * </dl> */ public interface IdempotentTask<T extends TaskContext> { /** - * A short id of the task to e.g. identify the task in the log. + * <p>A short id of the task to e.g. identify the task in the log.</p> * - * Prefer PascalCase and without white-space. + * <p>Prefer PascalCase and without white-space.</p> * - * Example: "EnableDocker" + * <p>Example: "EnableDocker"</p> */ default String name() { return getClass().getSimpleName(); } /** - * Execute an administrative task to converge towards some ideal state, whether it is - * system state or in-memory Java state. + * <p>Execute an administrative task to converge towards some ideal state, whether it is + * system state or in-memory Java state.</p> * - * converge() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`. + * <p>converge() must be idempotent: it may be called any number of times, or + * interrupted at any time e.g. by `kill -9`.</p> * - * converge() is not thread safe: The caller must ensure there is at most one invocation - * of converge() at any given time. + * <p>converge() is not thread safe: The caller must ensure there is at most one invocation + * of converge() at any given time.</p> * * @return false if already converged, i.e. was a no-op. A typical sequence of converge() * calls on a IdempotentTask will consist of: @@ -38,4 +40,22 @@ public interface IdempotentTask<T extends TaskContext> { * @throws RuntimeException (or a subclass) if the task is unable to converge. */ boolean converge(T context); + + /** + * <p>Converge the task towards some state where it can be suspended. The + * TaskContext should provide enough to determine what kind of suspend is wanted, e.g. + * suspension of only the task, or the task and the resources/processes it manages.</p> + * + * <p>convergeSuspend() must be idempotent: it may be called any number of times, or + * interrupted at any time e.g. by `kill -9`.</p> + * + * <p>convergeSuspend() is not thread safe: The caller must ensure there is at most one + * invocation of convergeSuspend() at any given time.</p> + * + * @return false if already converged, i.e. was a no-op + * @throws RuntimeException (or a subclass) if the task is unable to suspend. + */ + default boolean convergeSuspend(T context) { + return false; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index be54df31c50..3b17972db6d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -5,12 +5,15 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class NodeAttributes { private Optional<Long> restartGeneration = Optional.empty(); private Optional<Long> rebootGeneration = Optional.empty(); private Optional<DockerImage> dockerImage = Optional.empty(); + private Optional<String> vespaVersion = Optional.empty(); private Optional<String> hardwareDivergence = Optional.empty(); public NodeAttributes() { } @@ -30,6 +33,11 @@ public class NodeAttributes { return this; } + public NodeAttributes withVespaVersion(String vespaVersion) { + this.vespaVersion = Optional.of(vespaVersion); + return this; + } + public NodeAttributes withHardwareDivergence(String hardwareDivergence) { this.hardwareDivergence = Optional.of(hardwareDivergence); return this; @@ -48,13 +56,17 @@ public class NodeAttributes { return dockerImage; } + public Optional<String> getVespaVersion() { + return vespaVersion; + } + public Optional<String> getHardwareDivergence() { return hardwareDivergence; } @Override public int hashCode() { - return Objects.hash(restartGeneration, rebootGeneration, dockerImage, hardwareDivergence); + return Objects.hash(restartGeneration, rebootGeneration, dockerImage, vespaVersion, hardwareDivergence); } @Override @@ -67,16 +79,20 @@ public class NodeAttributes { return Objects.equals(restartGeneration, other.restartGeneration) && Objects.equals(rebootGeneration, other.rebootGeneration) && Objects.equals(dockerImage, other.dockerImage) + && Objects.equals(vespaVersion, other.vespaVersion) && Objects.equals(hardwareDivergence, other.hardwareDivergence); } @Override public String toString() { - return "NodeAttributes{" + - "restartGeneration=" + restartGeneration.map(String::valueOf).orElse("") + - ", rebootGeneration=" + rebootGeneration.map(String::valueOf).orElse("") + - ", dockerImage=" + dockerImage.map(DockerImage::asString).orElse("") + - ", hardwareDivergence='" + hardwareDivergence.orElse(null) + "'" + - '}'; + return Stream.of( + restartGeneration.map(gen -> "restartGeneration=" + gen), + rebootGeneration.map(gen -> "rebootGeneration=" + gen), + dockerImage.map(img -> "dockerImage=" + img.asString()), + vespaVersion.map(ver -> "vespaVersion=" + ver), + hardwareDivergence.map(hwDivg -> "hardwareDivergence=" + hwDivg)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.joining(", ", "NodeAttributes{", "}")); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 71229818975..fb2a1e3f890 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -207,6 +207,7 @@ public class RealNodeRepository implements NodeRepository { node.currentDockerImage = nodeAttributes.getDockerImage().map(DockerImage::asString).orElse(null); node.currentRestartGeneration = nodeAttributes.getRestartGeneration().orElse(null); node.currentRebootGeneration = nodeAttributes.getRebootGeneration().orElse(null); + node.vespaVersion = nodeAttributes.getVespaVersion().orElse(null); node.hardwareDivergence = nodeAttributes.getHardwareDivergence().orElse(null); return node; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index ef5f2e60220..8cd877b25e5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -91,6 +91,14 @@ public class DockerOperationsImpl implements DockerOperations { command.withVolume("/var/lib/sia", "/var/lib/sia"); } + // TODO When rolling out host-admin on-prem: Always map in /var/zpe from host + make sure zpu is configured on host + if (environment.getCloud().equalsIgnoreCase("yahoo")) { + Path pathInNode = environment.pathInNodeUnderVespaHome("var/zpe"); + command.withVolume(environment.pathInHostFromPathInNode(containerName, pathInNode).toString(), pathInNode.toString()); + } else if (environment.getNodeType() == NodeType.host) { + command.withVolume("/var/zpe", environment.pathInNodeUnderVespaHome("var/zpe").toString()); + } + if (environment.getNodeType() == NodeType.proxyhost) { command.withVolume("/opt/yahoo/share/ssl/certs/", "/opt/yahoo/share/ssl/certs/"); } @@ -354,7 +362,6 @@ public class DockerOperationsImpl implements DockerOperations { directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/yca"), true); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/ycore++"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/zookeeper"), false); - directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/zpe"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("tmp"), false); directoriesToMount.put(environment.pathInNodeUnderVespaHome("var/container-data"), false); if (environment.getNodeType() == NodeType.proxyhost) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 0c5dd72c968..7f2d1f1eff7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -5,16 +5,19 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.collections.Pair; +import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; +import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; @@ -46,6 +49,7 @@ public class StorageMaintainer { private static final ContainerName NODE_ADMIN = new ContainerName("node-admin"); private static final ObjectMapper objectMapper = new ObjectMapper(); + private final GaugeWrapper numberOfCoredumpsOnHost; private final CounterWrapper numberOfNodeAdminMaintenanceFails; private final DockerOperations dockerOperations; private final ProcessExecuter processExecuter; @@ -54,7 +58,6 @@ public class StorageMaintainer { private final Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>(); - public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { this.dockerOperations = dockerOperations; this.processExecuter = processExecuter; @@ -63,44 +66,98 @@ public class StorageMaintainer { Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails"); + numberOfCoredumpsOnHost = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.coredumps"); } public void writeMetricsConfig(ContainerName containerName, NodeSpec node) { - final Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode( - containerName, Paths.get("/etc/yamas-agent/")); - - Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); - SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all") - .withTag("parentHostname", environment.getParentHostHostname()); + List<SecretAgentCheckConfig> configs = new ArrayList<>(); + // host-life Path hostLifeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); - SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath) - .withTag("namespace", "Vespa") + SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath); + configs.add(annotatedCheck(node, hostLifeSchedule)); + + // ntp + Path ntpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp"); + SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath); + configs.add(annotatedCheck(node, ntpSchedule)); + + // coredumps (except for the done coredumps which is handled by the host) + Path coredumpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); + SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300, + coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin", + "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString()); + configs.add(annotatedCheck(node, coredumpSchedule)); + + if (node.getNodeType() != NodeType.config) { + // vespa-health + Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); + SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all"); + configs.add(annotatedCheck(node, vespaHealthSchedule)); + + // vespa + Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); + SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); + configs.add(annotatedCheck(node, vespaSchedule)); + } + + if (node.getNodeType() == NodeType.config) { + // configserver + Path configServerCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2"); + SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60, + configServerCheckPath, "-zero", "configserver"); + configs.add(annotatedCheck(node, configServerSchedule)); + + //zkbackupage + Path zkbackupCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); + SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300, + zkbackupCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), + "-m", "150", "-a", "config-zkbackupage"); + configs.add(annotatedCheck(node, zkbackupSchedule)); + } + + if (node.getNodeType() == NodeType.proxy) { + //routing-configage + Path routingAgeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); + SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, + routingAgeCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(), + "-m", "90", "-a", "routing-configage"); + configs.add(annotatedCheck(node, routingAgeSchedule)); + + //ssl-check + Path sslCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); + SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300, + sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30"); + configs.add(annotatedCheck(node, sslSchedule)); + } + + // Write config and restart yamas-agent + Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/etc/yamas-agent/")); + configs.forEach(s -> IOExceptionUtil.uncheck(() -> s.writeTo(yamasAgentFolder))); + final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; + dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); + } + + private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) { + check.withTag("namespace", "Vespa") .withTag("role", "tenants") .withTag("flavor", node.getFlavor()) .withTag("canonicalFlavor", node.getCanonicalFlavor()) .withTag("state", node.getState().toString()) .withTag("zone", environment.getZone()) .withTag("parentHostname", environment.getParentHostHostname()); - node.getOwner().ifPresent(owner -> hostLifeSchedule + node.getOwner().ifPresent(owner -> check .withTag("tenantName", owner.getTenant()) .withTag("app", owner.getApplication() + "." + owner.getInstance()) .withTag("applicationName", owner.getApplication()) .withTag("instanceName", owner.getInstance()) .withTag("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance())); - node.getMembership().ifPresent(membership -> hostLifeSchedule + node.getMembership().ifPresent(membership -> check .withTag("clustertype", membership.getClusterType()) .withTag("clusterid", membership.getClusterId())); - node.getVespaVersion().ifPresent(version -> hostLifeSchedule.withTag("vespaVersion", version)); + node.getVespaVersion().ifPresent(version -> check.withTag("vespaVersion", version)); - try { - vespaSchedule.writeTo(yamasAgentFolder); - hostLifeSchedule.writeTo(yamasAgentFolder); - final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; - dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); - } catch (IOException e) { - throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); - } + return check; } public void writeFilebeatConfig(ContainerName containerName, NodeSpec node) { @@ -218,6 +275,14 @@ public class StorageMaintainer { * @param force Set to true to bypass throttling */ public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node, boolean force) { + // Sample number of coredumps on the host + try { + numberOfCoredumpsOnHost.sample(Files.list(environment.pathInNodeAdminToDoneCoredumps()).count()); + } catch (IOException e) { + // Ignore for now - this is either test or a misconfiguration + } + + // Return early if throttled if (! getMaintenanceThrottlerFor(containerName).shouldHandleCoredumpsNow() && !force) return; MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 869e59d890b..f7e9c3ca1d8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -1,14 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.identity; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -49,15 +47,12 @@ import static java.util.Collections.singleton; * * @author bjorncs */ -@SuppressWarnings("deprecation") // TODO Use new entity response types public class AthenzCredentialsMaintainer { private static final Duration EXPIRY_MARGIN = Duration.ofDays(1); private static final Duration REFRESH_PERIOD = Duration.ofDays(1); private static final Path CONTAINER_SIA_DIRECTORY = Paths.get("/var/lib/sia"); - private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); - private final boolean enabled; private final PrefixLogger log; private final String hostname; @@ -176,7 +171,7 @@ public class AthenzCredentialsMaintainer { } private boolean isCertificateExpired(Instant expiry, Instant now) { - return expiry.minus(EXPIRY_MARGIN).isAfter(now); + return now.isAfter(expiry.minus(EXPIRY_MARGIN)); } private void registerIdentity(VespaUniqueInstanceId instanceId, Set<String> ipAddresses) { @@ -189,7 +184,7 @@ public class AthenzCredentialsMaintainer { configserverIdentity, containerIdentity, instanceId.asDottedString(), - toAttestationDataString(signedIdentityDocument), + EntityBindingsMapper.toAttestationData(signedIdentityDocument), false, csr); writePrivateKeyAndCertificate(keyPair.getPrivate(), instanceIdentity.certificate()); @@ -243,30 +238,6 @@ public class AthenzCredentialsMaintainer { } // TODO Move to vespa-athenz - private String toAttestationDataString(SignedIdentityDocument signedIdDoc) throws JsonProcessingException { - com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument idDoc = signedIdDoc.identityDocument(); - com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument identityDocumentPayload = - new com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument( - com.yahoo.vespa.athenz.identityprovider.api.bindings.ProviderUniqueId.fromVespaUniqueInstanceId(idDoc.providerUniqueId()), - idDoc.configServerHostname(), - idDoc.instanceHostname(), - idDoc.createdAt(), - idDoc.ipAddresses()); - String rawIdentityDocument = objectMapper.writeValueAsString(identityDocumentPayload); - com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument payload = - new com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument( - rawIdentityDocument, - signedIdDoc.signature(), - signedIdDoc.signingKeyVersion(), - signedIdDoc.providerUniqueId().asDottedString(), - signedIdDoc.dnsSuffix(), - signedIdDoc.providerService().getFullName(), - signedIdDoc.ztsEndpoint(), - signedIdDoc.documentVersion()); - return objectMapper.writeValueAsString(payload); - } - - // TODO Move to vespa-athenz private static Path getPrivateKeyFile(Path root, AthenzService service) { return root .resolve("keys") diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java index 98394f52857..427588c287d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImpl.java @@ -43,6 +43,7 @@ import static com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater.S */ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); + static final String TRANSITION_EXCEPTION_MESSAGE = "NodeAdminStateUpdater has not run since current wanted state was set"; private final AtomicBoolean terminated = new AtomicBoolean(false); private State currentState = SUSPENDED_NODE_ADMIN; @@ -50,6 +51,7 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { private boolean workToDoNow = true; private final Object monitor = new Object(); + private RuntimeException lastConvergenceException; private final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); private final ScheduledExecutorService specVerifierScheduler = @@ -143,15 +145,19 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { } @Override - public boolean setResumeStateAndCheckIfResumed(State wantedState) { + public void setResumeStateAndCheckIfResumed(State wantedState) { synchronized (monitor) { if (this.wantedState != wantedState) { log.info("Wanted state change: " + this.wantedState + " -> " + wantedState); this.wantedState = wantedState; + setLastConvergenceException(null); signalWorkToBeDone(); } - return currentState == wantedState; + if (currentState != wantedState) { + throw Optional.ofNullable(lastConvergenceException) + .orElseGet(() -> new RuntimeException(TRANSITION_EXCEPTION_MESSAGE)); + } } } @@ -187,9 +193,12 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { try { convergeState(wantedStateCopy); + setLastConvergenceException(null); } catch (OrchestratorException | ConvergenceException | HttpException e) { + setLastConvergenceException(e); log.info("Unable to converge to " + wantedStateCopy + ": " + e.getMessage()); - } catch (Exception e) { + } catch (RuntimeException e) { + setLastConvergenceException(e); log.log(LogLevel.ERROR, "Error while trying to converge to " + wantedStateCopy, e); } @@ -206,6 +215,12 @@ public class NodeAdminStateUpdaterImpl implements NodeAdminStateUpdater { fetchContainersToRunFromNodeRepository(); } + private void setLastConvergenceException(RuntimeException exception) { + synchronized (monitor) { + lastConvergenceException = exception; + } + } + /** * This method attempts to converge node-admin w/agents to a {@link State} * with respect to: freeze, Orchestrator, and services running. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java index 841f464e014..8a926b511e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminStateUpdater.java @@ -8,9 +8,11 @@ public interface NodeAdminStateUpdater extends NodeAdminDebugHandler { enum State { TRANSITIONING, RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED} /** - * Set the wanted state, and return whether the current state equals it. + * Set the wanted state, and assert whether the current state equals it. * Typically, this method should be called repeatedly until current state * has converged. + * + * @throws RuntimeException (or a subclass) if the state has not converged yet. */ - boolean setResumeStateAndCheckIfResumed(State wantedState); + void setResumeStateAndCheckIfResumed(State wantedState); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java index a8dcde02ca6..8411df09c70 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; +import com.yahoo.yolean.Exceptions; import javax.inject.Inject; import javax.ws.rs.core.MediaType; @@ -94,9 +95,13 @@ public class RestApiHandler extends LoggingRequestHandler{ } if (wantedState != null) { - return refresher.setResumeStateAndCheckIfResumed(wantedState) ? - new SimpleResponse(200, "ok") : - new SimpleResponse(409, "fail"); + try { + refresher.setResumeStateAndCheckIfResumed(wantedState); + return new SimpleResponse(200, "ok"); + } catch (RuntimeException e) { + return new SimpleResponse(409, "Failed to converge to " + wantedState + ": " + + Exceptions.toMessageString(e)); + } } return new SimpleResponse(400, "unknown path " + path); } @@ -107,7 +112,7 @@ public class RestApiHandler extends LoggingRequestHandler{ SimpleResponse(int code, String message) { super(code); ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.put("jsonMessage", message); + objectNode.put("message", message); this.jsonMessage = objectNode.toString(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java index 24736683845..26ca069aceb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/IOExceptionUtil.java @@ -7,6 +7,10 @@ import java.nio.file.NoSuchFileException; import java.util.Optional; /** + * Utils related to IOException. + * + * todo: replace much of the below with com.yahoo.yolean.Exceptions::uncheck + * * @author hakonhall */ public class IOExceptionUtil { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index ed315708b51..8987a7d6af3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -55,10 +55,10 @@ public class MultiDockerTest { "DeleteContainerStorage with ContainerName { name=host2 }"); dockerTester.callOrderVerifier.assertInOrder( - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, hardwareDivergence='null'}", - "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, hardwareDivergence='null'}", + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}", + "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2}", "setNodeState host2.test.yahoo.com to ready", - "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, hardwareDivergence='null'}"); + "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index a83efc03fbe..db14efdd5d2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -42,8 +42,8 @@ public class RebootTest { "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null, dockerImage=dockerImage, vespaVersion='null'}"); NodeAdminStateUpdaterImpl updater = dockerTester.nodeAdminStateUpdater; - assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), - is(Optional.of("Not all node agents are frozen."))); +// assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), +// is(Optional.of("Not all node agents are frozen."))); updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 651da7caec5..fb841f63f0c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -30,7 +30,7 @@ public class RestartTest { // Check that the container is started and NodeRepo has received the PATCH update dockerTester.callOrderVerifier.assertInOrder( "createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, hardwareDivergence='null'}"); + "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3}"); wantedRestartGeneration = 2; currentRestartGeneration = 1; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 607dc080a90..02baf5959c9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -19,8 +19,9 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl.TRANSITION_EXCEPTION_MESSAGE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; @@ -62,7 +63,7 @@ public class NodeAdminStateUpdaterImplTest { suspendHostnames.add(parentHostname); // Initially everything is frozen to force convergence - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze @@ -70,35 +71,36 @@ public class NodeAdminStateUpdaterImplTest { verify(orchestrator, times(1)).resume(parentHostname); // Everything is running and we want to continue running, therefore we have converged - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); tickAfter(35); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(refresher, never()).signalWorkToBeDone(); // No attempt in changing state verify(orchestrator, times(1)).resume(parentHostname); // Already resumed // Lets try to suspend node admin only, immediately we get false back, and need to wait until next // tick before any change can happen - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); verify(refresher, times(1)).signalWorkToBeDone(); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); // Still no change + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); // Still no change verify(refresher, times(1)).signalWorkToBeDone(); // We already notified of work, dont need to do it again when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); tickAfter(0); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); verify(refresher, times(1)).signalWorkToBeDone(); // No change in desired state // First orchestration failure happens within the freeze convergence timeout, // and so should not call setFrozen(false) + final String exceptionMessage = "Cannot allow to suspend because some reason"; verify(nodeAdmin, times(1)).setFrozen(eq(false)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")) + doThrow(new RuntimeException(exceptionMessage)) .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); verify(nodeAdmin, times(1)).setFrozen(eq(false)); @@ -106,22 +108,22 @@ public class NodeAdminStateUpdaterImplTest { // and so SHOULD call setFrozen(false) when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(NodeAdminStateUpdaterImpl.FREEZE_CONVERGENCE_TIMEOUT.plusMinutes(1)); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")).doNothing() + doThrow(new RuntimeException(exceptionMessage)).doNothing() .when(orchestrator).suspend(eq(parentHostname)); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMessage); verify(refresher, times(1)).signalWorkToBeDone(); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // +1, since freeze convergence have timed out tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // At this point orchestrator will say its OK to suspend, but something goes wrong when we try to stop services verify(orchestrator, times(0)).suspend(eq(parentHostname), eq(suspendHostnames)); doThrow(new RuntimeException("Failed to stop services")).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); // Change in wanted state, no need to wait verify(orchestrator, times(1)).suspend(eq(parentHostname), eq(suspendHostnames)); verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state @@ -130,58 +132,69 @@ public class NodeAdminStateUpdaterImplTest { // Finally we are successful in transitioning to frozen tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); // We are in desired state, no changes will happen reset(nodeAdmin); tickAfter(35); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); verify(refresher, times(2)).signalWorkToBeDone(); // No change in desired state verifyNoMoreInteractions(nodeAdmin); // Lets try going back to resumed when(nodeAdmin.setFrozen(eq(false))).thenReturn(false).thenReturn(true); // NodeAgents not converged to yet - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "NodeAdmin is not yet unfrozen"); doThrow(new OrchestratorException("Cannot allow to suspend " + parentHostname)).doNothing() .when(orchestrator).resume(parentHostname); tickAfter(35); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, "Cannot allow to suspend basehost1.test.yahoo.com"); tickAfter(35); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); } @Test public void half_transition_revert() { + final String exceptionMsg = "Cannot allow to suspend because some reason"; mockNodeRepo(3); // Initially everything is frozen to force convergence when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); doNothing().when(orchestrator).resume(parentHostname); tickAfter(0); // The first tick should unfreeze - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(nodeAdmin, times(1)).setFrozen(eq(false)); // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); - doThrow(new RuntimeException("Cannot allow to suspend because some reason")) + doThrow(new RuntimeException(exceptionMsg)) .when(orchestrator).suspend(eq(parentHostname)); - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); verify(nodeAdmin, times(1)).setFrozen(eq(true)); + assertResumeStateError(NodeAdminStateUpdater.State.SUSPENDED_NODE_ADMIN, exceptionMsg); // We change our mind, want to remain resumed - assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + assertResumeStateError(NodeAdminStateUpdater.State.RESUMED, TRANSITION_EXCEPTION_MESSAGE); tickAfter(0); - assertTrue(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); + refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Make sure that we unfreeze! } + private void assertResumeStateError(NodeAdminStateUpdater.State targetState, String reason) { + try { + refresher.setResumeStateAndCheckIfResumed(targetState); + fail("Expected set resume state to fail with \"" + reason + "\", but it succeeded without error"); + } catch (RuntimeException e) { + assertEquals(reason, e.getMessage()); + } + } + private void mockNodeRepo(int numberOfNodes) { List<NodeSpec> containersToRun = IntStream.range(0, numberOfNodes) .mapToObj(i -> new NodeSpec.Builder() diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java index fe09f23d282..fef5a695db6 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoreCollector.java @@ -157,7 +157,7 @@ public class CoreCollector { private Path compressCoredump(Path coredumpPath) throws IOException { if (! coredumpPath.toString().endsWith(".lz4")) { processExecuter.exec( - new String[]{LZ4_PATH, coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); + new String[]{LZ4_PATH, "-f", coredumpPath.toString(), coredumpPath.toString() + ".lz4"}); return coredumpPath; } else { @@ -167,7 +167,7 @@ public class CoreCollector { Path decompressedPath = Paths.get(coredumpPath.toString().replaceFirst("\\.lz4$", "")); Pair<Integer, String> result = processExecuter.exec( - new String[]{LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); + new String[] {LZ4_PATH, "-f", "-d", coredumpPath.toString(), decompressedPath.toString()}); if (result.getFirst() != 0) { throw new RuntimeException("Failed to decompress file " + coredumpPath + ": " + result); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 4497f55cb85..62e954afba3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -65,7 +66,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig) { - DefaultTimes defaults = new DefaultTimes(zone.environment()); + DefaultTimes defaults = new DefaultTimes(zone); jobControl = new JobControl(nodeRepository.database()); infrastructureVersions = new InfrastructureVersions(nodeRepository.database()); @@ -151,7 +152,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; - DefaultTimes(Environment environment) { + DefaultTimes(Zone zone) { failGrace = Duration.ofMinutes(60); periodicRedeployInterval = Duration.ofMinutes(30); operatorChangeRedeployInterval = Duration.ofMinutes(1); @@ -164,13 +165,14 @@ public class NodeRepositoryMaintenance extends AbstractComponent { infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; + Environment environment = zone.environment(); if (environment.isTest()) retiredExpiry = Duration.ofMinutes(1); // fast turnaround as test envs don't have persistent data else retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days - if (environment.equals(Environment.prod)) { + if (environment.equals(Environment.prod) && zone.system() == SystemName.main) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredInterval = Duration.ofMinutes(29); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index 12a0496ed2e..203f52a7b70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -38,6 +38,8 @@ public class CuratorDatabase { /** Whether we should return data from the cache or always read fro ZooKeeper */ private final boolean useCache; + private final Object cacheCreationLock = new Object(); + /** * All keys, to allow reentrancy. * This will grow forever with the number of applications seen, but this should be too slow to be a problem. @@ -94,14 +96,42 @@ public class CuratorDatabase { public Optional<byte[]> getData(Path path) { return getCache().getData(path); } + private static class CacheAndGeneration { + public CacheAndGeneration(CuratorDatabaseCache cache, long generation) + { + this.cache = cache; + this.generation = generation; + } + public boolean expired() { + return generation != cache.generation(); + } + public CuratorDatabaseCache validCache() { + if (expired()) { + throw new IllegalStateException("The cache has generation " + cache.generation() + + " while the root genration counter in zookeeper says " + generation + + ". That is totally unacceptable and must be a sever programming error in my close vicinity."); + } + return cache; + } + + private CuratorDatabaseCache cache; + private long generation; + } + private CacheAndGeneration getCacheSnapshot() { + return new CacheAndGeneration(cache.get(), changeGenerationCounter.get()); + } private CuratorDatabaseCache getCache() { - CuratorDatabaseCache cache = this.cache.get(); - long currentCuratorGeneration = changeGenerationCounter.get(); - if (currentCuratorGeneration != cache.generation()) { // current cache is invalid - start new - cache = newCache(currentCuratorGeneration); - this.cache.set(cache); + CacheAndGeneration cacheAndGeneration = getCacheSnapshot(); + while (cacheAndGeneration.expired()) { + synchronized (cacheCreationLock) { // Prevent a race for creating new caches + cacheAndGeneration = getCacheSnapshot(); + if (cacheAndGeneration.expired()) { + cache.set(newCache(changeGenerationCounter.get())); + cacheAndGeneration = getCacheSnapshot(); + } + } } - return cache; + return cacheAndGeneration.validCache(); } /** Caches must only be instantiated using this method */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index fd9ee237046..31d9a606d91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -111,6 +111,7 @@ public class NodePatcher { .map(DockerImage::tagAsVersion) .orElse(Version.emptyVersion); return node.with(node.status().withVespaVersion(versionFromImage)); + case "vespaVersion" : case "currentVespaVersion" : return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); case "failCount" : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java index 6d92f9c4541..bea7973541a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2.filter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -11,6 +13,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.security.cert.X509Certificate; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; @@ -31,9 +34,13 @@ class NodeIdentifier { private final Zone zone; private final NodeRepository nodeRepository; + + private final Supplier<List<Node>> nodeCache; + NodeIdentifier(Zone zone, NodeRepository nodeRepository) { this.zone = zone; this.nodeRepository = nodeRepository; + nodeCache = Suppliers.memoizeWithExpiration(nodeRepository::getNodes, 1, TimeUnit.MINUTES); } NodePrincipal resolveNode(List<X509Certificate> certificateChain) throws NodeIdentifierException { @@ -73,7 +80,7 @@ class NodeIdentifier { private String getHostFromCalypsoCertificate(List<SubjectAlternativeName> sans) { String openstackId = getUniqueInstanceId(sans); - return nodeRepository.getNodes().stream() + return nodeCache.get().stream() .filter(node -> node.openStackId().equals(openstackId)) .map(Node::hostname) .findFirst() @@ -72,6 +72,7 @@ <module>documentapi</module> <module>document</module> <module>documentgen-test</module> + <module>fat-model-dependencies</module> <module>fileacquirer</module> <module>filedistribution</module> <module>fsa</module> diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h index 4be1f00dcbc..d18c4840009 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h @@ -5,18 +5,12 @@ #include <vespa/searchcommon/common/range.h> #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace fef { - class TermFieldMatchData; -} -namespace queryeval { - class SearchIterator; -} +namespace search::fef { class TermFieldMatchData; } +namespace search::queryeval { class SearchIterator; } +namespace search { class QueryTermBase; } -class QueryTermBase; - -namespace attribute { +namespace search::attribute { class ISearchContext { public: @@ -24,8 +18,8 @@ public: using DocId = uint32_t; private: - virtual bool onCmp(DocId docId, int32_t &weight) const = 0; - virtual bool onCmp(DocId docId) const = 0; + virtual int32_t onFind(DocId docId, int32_t elementId, int32_t &weight) const = 0; + virtual int32_t onFind(DocId docId, int32_t elementId) const = 0; public: virtual ~ISearchContext() {} @@ -57,10 +51,19 @@ public: virtual const QueryTermBase &queryTerm() const = 0; virtual const vespalib::string &attributeName() const = 0; - bool cmp(DocId docId, int32_t &weight) const { return onCmp(docId, weight); } - bool cmp(DocId docId) const { return onCmp(docId); } + int32_t find(DocId docId, int32_t elementId, int32_t &weight) const { return onFind(docId, elementId, weight); } + int32_t find(DocId docId, int32_t elementId) const { return onFind(docId, elementId); } + bool matches(DocId docId, int32_t &weight) const { + weight = 0; + int32_t oneWeight(0); + int32_t firstId = find(docId, 0, oneWeight); + for (int32_t id(firstId); id >= 0; id = find(docId, id + 1, oneWeight)) { + weight += oneWeight; + } + return firstId >= 0; + } + bool matches(DocId doc) const { return find(doc, 0) >= 0; } }; } -} diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 36e76b02b0b..b1570c10221 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -68,6 +68,7 @@ vespa_define_module( src/tests/proton/attribute/attribute_populator src/tests/proton/attribute/attribute_usage_filter src/tests/proton/attribute/attributes_state_explorer + src/tests/proton/attribute/document_field_extractor src/tests/proton/attribute/document_field_populator src/tests/proton/attribute/exclusive_attribute_read_accessor src/tests/proton/attribute/imported_attributes_context diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index ee5f29255fb..634f69a3820 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_test"); #include <vespa/config-attributes.h> #include <vespa/document/fieldvalue/document.h> @@ -15,6 +13,7 @@ LOG_SETUP("attribute_test"); #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -44,6 +43,9 @@ LOG_SETUP("attribute_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_test"); + namespace vespa { namespace config { namespace search {}}} using namespace config; @@ -139,6 +141,7 @@ struct Fixture : Fixture(1) { } + ~Fixture(); void allocAttributeWriter() { _aw = std::make_unique<AttributeWriter>(_m); } @@ -155,8 +158,12 @@ struct Fixture _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); } void update(SerialNum serialNum, const DocumentUpdate &upd, + DocumentIdT lid, bool immediateCommit, IFieldUpdateCallback & onUpdate) { + _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback, onUpdate); + } + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit) { - _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback); + _aw->update(serialNum, doc, lid, immediateCommit, emptyCallback); } void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) { _aw->remove(serialNum, lid, immediateCommit, emptyCallback); @@ -172,6 +179,7 @@ struct Fixture } }; +Fixture::~Fixture() = default; TEST_F("require that attribute writer handles put", Fixture) { @@ -442,8 +450,9 @@ TEST_F("require that attribute writer handles update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10))); + DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + f.update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); @@ -453,9 +462,9 @@ TEST_F("require that attribute writer handles update", Fixture) EXPECT_EQUAL(1u, ibuf.size()); EXPECT_EQUAL(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit); // same sync token as previous + f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit); // lower sync token than previous + f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -488,7 +497,8 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } @@ -675,7 +685,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); @@ -773,6 +784,158 @@ TEST_F("require that AttributeWriter::forceCommit() clears search cache in impor EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } +struct StructFixtureBase : public Fixture +{ + DocumentType _type; + const Field _valueField; + StructDataType _structFieldType; + + StructFixtureBase() + : Fixture(), + _type("test"), + _valueField("value", 2, *DataType::INT, true), + _structFieldType("struct") + { + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + _type.addField(_valueField); + _structFieldType.addField(_valueField); + } + ~StructFixtureBase(); + + std::unique_ptr<StructFieldValue> + makeStruct() + { + return std::make_unique<StructFieldValue>(_structFieldType); + } + + std::unique_ptr<StructFieldValue> + makeStruct(const int32_t value) + { + auto ret = makeStruct(); + ret->setValue(_valueField, IntFieldValue(value)); + return ret; + } + + std::unique_ptr<Document> + makeDoc() + { + return std::make_unique<Document>(_type, DocumentId("id::test::1")); + } +}; + +StructFixtureBase::~StructFixtureBase() = default; + +struct StructArrayFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const ArrayDataType _structArrayFieldType; + const Field _structArrayField; + + StructArrayFixture() + : StructFixtureBase(), + _structArrayFieldType(_structFieldType), + _structArrayField("array", _structArrayFieldType, true) + { + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structArrayField); + } + ~StructArrayFixture(); + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + ArrayFieldValue s(_structArrayFieldType); + for (const auto &arrayValue : arrayValues) { + s.add(*makeStruct(arrayValue)); + } + doc->setValue(_structArrayField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); + EXPECT_EQUAL(value, valueAttr->getInt(lid)); + attribute::IntegerContent ibuf; + ibuf.fill(*arrayValueAttr, lid); + EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + for (size_t i = 0; i < arrayValues.size(); ++i) { + EXPECT_EQUAL(arrayValues[i], ibuf[i]); + } + } +}; + +StructArrayFixture::~StructArrayFixture() = default; + +TEST_F("require that update with doc argument updates compound attributes (array)", StructArrayFixture) +{ + auto doc = f.makeDoc(10, {11, 12}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {11, 12})); + doc = f.makeDoc(20, {21}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {21})); +} + +struct StructMapFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const MapDataType _structMapFieldType; + const Field _structMapField; + + StructMapFixture() + : StructFixtureBase(), + _structMapFieldType(*DataType::INT, _structFieldType), + _structMapField("map", _structMapFieldType, true) + { + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structMapField); + } + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + MapFieldValue s(_structMapFieldType); + for (const auto &mapValue : mapValues) { + s.put(IntFieldValue(mapValue.first), *makeStruct(mapValue.second)); + } + doc->setValue(_structMapField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); + auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); + EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + attribute::IntegerContent mapKeys; + mapKeys.fill(*mapKeyAttr, lid); + attribute::IntegerContent mapValues; + mapValues.fill(*mapValueAttr, lid); + EXPECT_EQUAL(expMap.size(), mapValues.size()); + EXPECT_EQUAL(expMap.size(), mapKeys.size()); + size_t i = 0; + for (const auto &expMapElem : expMap) { + EXPECT_EQUAL(expMapElem.first, mapKeys[i]); + EXPECT_EQUAL(expMapElem.second, mapValues[i]); + ++i; + } + } +}; + +TEST_F("require that update with doc argument updates compound attributes (map)", StructMapFixture) +{ + auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); + doc = f.makeDoc(20, {{42, 21}}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/document_field_extractor/CMakeLists.txt new file mode 100644 index 00000000000..7d7b798febe --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_document_field_extractor_test_app TEST + SOURCES + document_field_extractor_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_document_field_extractor_test_app COMMAND searchcore_document_field_extractor_test_app) diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp new file mode 100644 index 00000000000..bad27938d4b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -0,0 +1,347 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/documentid.h> +#include <vespa/document/base/exceptions.h> +#include <vespa/document/base/field.h> +#include <vespa/document/base/fieldpath.h> +#include <vespa/document/datatype/datatypes.h> +#include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/mapfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/fieldvalue/structfieldvalue.h> +#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/searchcore/proton/attribute/document_field_extractor.h> +#include <vespa/vespalib/testkit/testapp.h> + +using document::Field; +using document::DataType; +using document::DocumentType; +using document::StructDataType; +using document::ArrayDataType; +using document::WeightedSetDataType; +using document::MapDataType; +using document::StructFieldValue; +using document::ArrayFieldValue; +using document::WeightedSetFieldValue; +using document::IntFieldValue; +using document::StringFieldValue; +using document::MapFieldValue; +using document::Document; +using document::DocumentId; +using document::FieldPath; +using document::FieldValue; +using document::FieldNotFoundException; +using proton::DocumentFieldExtractor; + +namespace +{ + +const ArrayDataType arrayTypeInt(*DataType::INT); +const ArrayDataType arrayTypeString(*DataType::STRING); +const WeightedSetDataType weightedSetTypeInt(*DataType::INT, false, false); +const WeightedSetDataType weightedSetTypeString(*DataType::STRING, false, false); +const int32_t noInt(search::attribute::getUndefined<int32_t>()); +const vespalib::string noString(""); + +std::unique_ptr<FieldValue> +makeIntArray(const std::vector<int32_t> &array) +{ + auto result = std::make_unique<ArrayFieldValue>(arrayTypeInt); + for (const auto &elem : array) { + result->append(std::make_unique<IntFieldValue>(elem)); + } + return result; +} + +std::unique_ptr<FieldValue> +makeStringArray(const std::vector<vespalib::string> &array) +{ + auto result = std::make_unique<ArrayFieldValue>(arrayTypeString); + for (const auto &elem : array) { + result->append(std::make_unique<StringFieldValue>(elem)); + } + return result; +} + +std::unique_ptr<FieldValue> +makeIntWeightedSet(const std::vector<std::pair<int32_t, int32_t>> &array) +{ + auto result = std::make_unique<WeightedSetFieldValue>(weightedSetTypeInt); + for (const auto &elem : array) { + result->add(IntFieldValue(elem.first), elem.second); + } + return result; +} + +std::unique_ptr<FieldValue> +makeStringWeightedSet(const std::vector<std::pair<vespalib::string, int32_t>> &array) +{ + auto result = std::make_unique<WeightedSetFieldValue>(weightedSetTypeString); + for (const auto &elem : array) { + result->add(StringFieldValue(elem.first), elem.second); + } + return result; +} + +} + +struct FixtureBase +{ + DocumentType type; + const Field weightField; + const Field nameField; + std::unique_ptr<Document> doc; + std::unique_ptr<DocumentFieldExtractor> extractor; + + FixtureBase(bool byteWeight) + : type("test"), + weightField("weight", 1, byteWeight ? *DataType::BYTE : *DataType::INT, true), + nameField("name", 2, *DataType::STRING, true), + doc(), + extractor() + { + } + + ~FixtureBase(); + + Document * + makeDoc() + { + doc = std::make_unique<Document>(type, DocumentId("id::test::1")); + extractor = std::make_unique<DocumentFieldExtractor>(*doc); + return doc.get(); + } + + FieldPath + makeFieldPath(const vespalib::string &path) + { + FieldPath fieldPath; + try { + type.buildFieldPath(fieldPath, path); + } catch (FieldNotFoundException &) { + fieldPath = FieldPath(); + } + if (!DocumentFieldExtractor::isSupported(fieldPath)) { + fieldPath = FieldPath(); + } + return fieldPath; + } + + void + assertExtracted(const vespalib::string &path, + std::unique_ptr<FieldValue> expected) { + FieldPath fieldPath(makeFieldPath(path)); + std::unique_ptr<FieldValue> fv = extractor->getFieldValue(fieldPath); + if (expected) { + ASSERT_TRUE(fv); + EXPECT_EQUAL(*expected, *fv); + } else { + EXPECT_TRUE(!fv); + } + } +}; + +FixtureBase::~FixtureBase() = default; + +struct SimpleFixture : public FixtureBase +{ + SimpleFixture(bool byteWeight = false) + : FixtureBase(byteWeight) + { + type.addField(weightField); + type.addField(nameField); + } +}; + +TEST_F("require that simple fields give simple values", SimpleFixture) +{ + auto doc = f.makeDoc(); + doc->setValue(f.weightField, IntFieldValue(200)); + doc->setValue(f.nameField, StringFieldValue("name200b")); + TEST_DO(f.assertExtracted("weight", std::make_unique<IntFieldValue>(200))); + TEST_DO(f.assertExtracted("name", std::make_unique<StringFieldValue>("name200b"))); +} + +struct ArrayFixture : public FixtureBase +{ + const ArrayDataType weightArrayFieldType; + const Field weightArrayField; + const ArrayDataType valueArrayFieldType; + const Field valueArrayField; + + ArrayFixture(bool byteWeight = false) + : FixtureBase(byteWeight), + weightArrayFieldType(weightField.getDataType()), + weightArrayField("weight", weightArrayFieldType, true), + valueArrayFieldType(nameField.getDataType()), + valueArrayField("val", valueArrayFieldType, true) + { + type.addField(weightArrayField); + type.addField(valueArrayField); + } + + ~ArrayFixture(); +}; + +ArrayFixture::~ArrayFixture() = default; + +TEST_F("require that array fields give array values", ArrayFixture) +{ + auto doc = f.makeDoc(); + doc->setValue(f.weightArrayField, *makeIntArray({ 300, 301 })); + doc->setValue(f.valueArrayField, *makeStringArray({"v500", "v502"})); + TEST_DO(f.assertExtracted("weight", makeIntArray({ 300, 301}))); + TEST_DO(f.assertExtracted("val", makeStringArray({"v500", "v502"}))); +} + +struct WeightedSetFixture : public FixtureBase +{ + const WeightedSetDataType weightWeightedSetFieldType; + const Field weightWeightedSetField; + const WeightedSetDataType valueWeightedSetFieldType; + const Field valueWeightedSetField; + + WeightedSetFixture(bool byteWeight = false) + : FixtureBase(byteWeight), + weightWeightedSetFieldType(weightField.getDataType(), false, false), + weightWeightedSetField("weight", weightWeightedSetFieldType, true), + valueWeightedSetFieldType(*DataType::STRING, false, false), + valueWeightedSetField("val", valueWeightedSetFieldType, true) + { + type.addField(weightWeightedSetField); + type.addField(valueWeightedSetField); + } + + ~WeightedSetFixture(); +}; + +WeightedSetFixture::~WeightedSetFixture() = default; + +TEST_F("require that weighted set fields give weighted set values", WeightedSetFixture) +{ + auto doc = f.makeDoc(); + doc->setValue(f.weightWeightedSetField, *makeIntWeightedSet({{400, 10}, { 401, 13}})); + doc->setValue(f.valueWeightedSetField, *makeStringWeightedSet({{"600", 17}, {"604", 19}})); + TEST_DO(f.assertExtracted("weight", makeIntWeightedSet({{ 400, 10}, {401, 13}}))); + TEST_DO(f.assertExtracted("val", makeStringWeightedSet({{"600", 17}, {"604", 19}}))); +} + +struct StructFixtureBase : public FixtureBase +{ + StructDataType structFieldType; + + StructFixtureBase(bool byteWeight) + : FixtureBase(byteWeight), + structFieldType("struct") + { + structFieldType.addField(weightField); + structFieldType.addField(nameField); + } + + std::unique_ptr<StructFieldValue> + makeStruct() + { + return std::make_unique<StructFieldValue>(structFieldType); + } + + std::unique_ptr<StructFieldValue> + makeStruct(int weight, const vespalib::string &value) + { + auto ret = makeStruct(); + ret->setValue(weightField, IntFieldValue(weight)); + ret->setValue(nameField, StringFieldValue(value)); + return ret; + } + + std::unique_ptr<StructFieldValue> + makeStruct(int weight) + { + auto ret = makeStruct(); + ret->setValue(weightField, IntFieldValue(weight)); + return ret; + } + + std::unique_ptr<StructFieldValue> + makeStruct(const vespalib::string &value) + { + auto ret = makeStruct(); + ret->setValue(nameField, StringFieldValue(value)); + return ret; + } +}; + +struct StructArrayFixture : public StructFixtureBase +{ + const ArrayDataType structArrayFieldType; + const Field structArrayField; + + StructArrayFixture(bool byteWeight = false) + : StructFixtureBase(byteWeight), + structArrayFieldType(structFieldType), + structArrayField("s", 11, structArrayFieldType, true) + { + type.addField(structArrayField); + } + + ~StructArrayFixture(); +}; + +StructArrayFixture::~StructArrayFixture() = default; + +TEST_F("require that struct array field gives array values", StructArrayFixture) +{ + auto doc = f.makeDoc(); + ArrayFieldValue structArrayFieldValue(f.structArrayFieldType); + structArrayFieldValue.add(*f.makeStruct(1, "name1")); + structArrayFieldValue.add(*f.makeStruct(2)); + structArrayFieldValue.add(*f.makeStruct("name3")); + doc->setValue(f.structArrayField, structArrayFieldValue); + TEST_DO(f.assertExtracted("s.weight", makeIntArray({ 1, 2, noInt }))); + TEST_DO(f.assertExtracted("s.name", makeStringArray({ "name1", noString, "name3" }))); +} + +struct StructMapFixture : public StructFixtureBase +{ + const MapDataType structMapFieldType; + const Field structMapField; + + StructMapFixture(bool byteWeight = false, bool byteKey = false) + : StructFixtureBase(byteWeight), + structMapFieldType(byteKey ? *DataType::BYTE : *DataType::STRING, structFieldType), + structMapField("s", 12, structMapFieldType, true) + { + type.addField(structMapField); + } + + ~StructMapFixture(); +}; + +StructMapFixture::~StructMapFixture() = default; + +TEST_F("require that struct map field gives array values", StructMapFixture) +{ + auto doc = f.makeDoc(); + MapFieldValue structMapFieldValue(f.structMapFieldType); + structMapFieldValue.put(StringFieldValue("m0"), *f.makeStruct(10, "name10")); + structMapFieldValue.put(StringFieldValue("m1"), *f.makeStruct(11)); + structMapFieldValue.put(StringFieldValue("m2"), *f.makeStruct("name12")); + structMapFieldValue.put(StringFieldValue("m3"), *f.makeStruct()); + doc->setValue(f.structMapField, structMapFieldValue); + TEST_DO(f.assertExtracted("s.key", makeStringArray({ "m0", "m1", "m2", "m3" }))); + TEST_DO(f.assertExtracted("s.value.weight", makeIntArray({ 10, 11, noInt, noInt }))); + TEST_DO(f.assertExtracted("s.value.name", makeStringArray({ "name10", noString, "name12", noString }))); +} + +TEST_F("require that unknown field gives null value", FixtureBase(false)) +{ + f.makeDoc(); + TEST_DO(f.assertExtracted("unknown", std::unique_ptr<FieldValue>())); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 5c0edce0b94..153b1ae2867 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -2,6 +2,7 @@ #include <vespa/persistence/spi/result.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -35,6 +36,7 @@ LOG_SETUP("feedhandler_test"); using document::BucketId; using document::Document; using document::DocumentId; +using document::DocumentType; using document::DocumentTypeRepo; using document::DocumentUpdate; using document::GlobalId; @@ -181,7 +183,9 @@ struct MyFeedView : public test::DummyFeedView { int prune_removed_count; int update_count; SerialNum update_serial; - MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr); + const DocumentType *documentType; + MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr, + const DocTypeName &docTypeName); ~MyFeedView() override; void resetPutLatch(uint32_t count) { putLatch.reset(new vespalib::CountDownLatch(count)); } void preparePut(PutOperation &op) override { @@ -203,6 +207,8 @@ struct MyFeedView : public test::DummyFeedView { if (usePutRdz) { putRdz.run(); } + EXPECT_EQUAL(_docTypeRepo.get(), putOp.getDocument()->getRepo()); + EXPECT_EQUAL(documentType, &putOp.getDocument()->getType()); ++put_count; put_serial = putOp.getSerialNum(); metaStore.allocate(putOp.getDocument()->getId().getGlobalId()); @@ -216,6 +222,7 @@ struct MyFeedView : public test::DummyFeedView { void handleUpdate(FeedToken token, const UpdateOperation &op) override { (void) token; + EXPECT_EQUAL(documentType, &op.getUpdate()->getType()); ++update_count; update_serial = op.getSerialNum(); } @@ -237,7 +244,7 @@ struct MyFeedView : public test::DummyFeedView { } }; -MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr) +MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr, const DocTypeName &docTypeName) : test::DummyFeedView(dtr), putRdz(), usePutRdz(false), @@ -250,7 +257,8 @@ MyFeedView::MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr) move_count(0), prune_removed_count(0), update_count(0), - update_serial(0) + update_serial(0), + documentType(dtr->getDocumentType(docTypeName.getName())) {} MyFeedView::~MyFeedView() {} @@ -294,6 +302,13 @@ struct DocumentContext { } }; +struct TwoFieldsSchemaContext : public SchemaContext { + TwoFieldsSchemaContext() + : SchemaContext() + { + addField("i2"); + } +}; struct UpdateContext { DocumentUpdate::SP update; @@ -433,7 +448,7 @@ struct FeedHandlerFixture owner(), _state(), replayConfig(), - feedView(schema.getRepo()), + feedView(schema.getRepo(), schema.getDocType()), _bucketDB(), _bucketDBHandler(_bucketDB), handler(writeService, tlsSpec, schema.getDocType(), _state, owner, @@ -714,15 +729,13 @@ TEST_F("require that update with same document type repo is ok", FeedHandlerFixt TEST_F("require that update with different document type repo can be ok", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i1", false, true); } TEST_F("require that update with different document type repo can be rejected", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i2", true, true); } @@ -733,18 +746,31 @@ TEST_F("require that update with same document type repo is ok, fallback to crea TEST_F("require that update with different document type repo can be ok, fallback to create document", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i1", false, false); } TEST_F("require that update with different document type repo can be rejected, preventing fallback to create document", FeedHandlerFixture) { - SchemaContext schema; - schema.addField("i2"); + TwoFieldsSchemaContext schema; checkUpdate(f, schema, "i2", true, false); } +TEST_F("require that put with different document type repo is ok", FeedHandlerFixture) +{ + TwoFieldsSchemaContext schema; + DocumentContext doc_context("doc:test:foo", *schema.builder); + auto op = std::make_unique<PutOperation>(doc_context.bucketId, + Timestamp(10), doc_context.doc); + FeedTokenContext token_context; + EXPECT_EQUAL(schema.getRepo().get(), op->getDocument()->getRepo()); + EXPECT_NOT_EQUAL(f.schema.getRepo().get(), op->getDocument()->getRepo()); + EXPECT_NOT_EQUAL(f.feedView.documentType, &op->getDocument()->getType()); + f.handler.performOperation(std::move(token_context.token), std::move(op)); + EXPECT_EQUAL(1, f.feedView.put_count); + EXPECT_EQUAL(1, f.tls_writer.store_count); +} + } // namespace TEST_MAIN() diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 00eb59f120a..5d040024e63 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/attribute/i_attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -316,21 +317,23 @@ struct MyAttributeWriter : public IAttributeWriter std::set<vespalib::string> _attrs; proton::IAttributeManager::SP _mgr; MyTracer &_tracer; + MyAttributeWriter(MyTracer &tracer); ~MyAttributeWriter(); - virtual std::vector<AttributeVector *> + + std::vector<AttributeVector *> getWritableAttributes() const override { return std::vector<AttributeVector *>(); } - virtual AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { + AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { if (_attrs.count(attrName) == 0) { return nullptr; } AttrMap::const_iterator itr = _attrMap.find(attrName); return ((itr == _attrMap.end()) ? nullptr : itr->second.get()); } - virtual void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _putSerial = serialNum; _putDocId = doc.getId(); _putLid = lid; @@ -339,8 +342,8 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(SerialNum serialNum, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void remove(SerialNum serialNum, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _removeSerial = serialNum; _removeLid = lid; _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); @@ -348,37 +351,45 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(const LidVector & lidsToRemove, SerialNum serialNum, - bool immediateCommit, OnWriteDoneType) override { + void remove(const LidVector & lidsToRemove, SerialNum serialNum, + bool immediateCommit, OnWriteDoneType) override { for (uint32_t lid : lidsToRemove) { LOG(info, "MyAttributeAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); _removes.push_back(lid); _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); } } - virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd, - DocumentIdT lid, bool, OnWriteDoneType) override { + void update(SerialNum serialNum, const document::DocumentUpdate &upd, + DocumentIdT lid, bool, OnWriteDoneType, IFieldUpdateCallback & onUpdate) override { _updateSerial = serialNum; _updateDocId = upd.getId(); _updateLid = lid; + for (const auto & fieldUpdate : upd.getUpdates()) { + search::AttributeVector * attr = getWritableAttribute(fieldUpdate.getField().getName()); + onUpdate.onUpdateField(fieldUpdate.getField().getName(), attr); + } } - virtual void heartBeat(SerialNum) override { ++_heartBeatCount; } - virtual void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override { + void update(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { (void) serialNum; + (void) doc; + (void) lid; + (void) immediateCommit; + } + void heartBeat(SerialNum) override { ++_heartBeatCount; } + void compactLidSpace(uint32_t wantedLidLimit, SerialNum ) override { _wantedLidLimit = wantedLidLimit; } - virtual const proton::IAttributeManager::SP &getAttributeManager() const override { + const proton::IAttributeManager::SP &getAttributeManager() const override { return _mgr; } void forceCommit(SerialNum serialNum, OnWriteDoneType) override { - (void) serialNum; ++_commitCount; + ++_commitCount; _tracer.traceCommit(attributeAdapterTypeName, serialNum); } - virtual void onReplayDone(uint32_t docIdLimit) override - { - (void) docIdLimit; - } + void onReplayDone(uint32_t ) override { } + bool getHasCompoundAttribute() const override { return false; } }; MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) @@ -396,7 +407,7 @@ MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) cfg3.setTensorType(ValueType::from_spec("tensor(x[10])")); _attrMap["a3"] = search::AttributeFactory::createAttribute("test3", cfg3); } -MyAttributeWriter::~MyAttributeWriter() {} +MyAttributeWriter::~MyAttributeWriter() = default; struct MyTransport : public feedtoken::ITransport { @@ -420,7 +431,7 @@ struct MyResultHandler : public IGenericResultHandler { vespalib::Gate _gate; MyResultHandler() : _gate() {} - virtual void handle(const storage::spi::Result &) override { + void handle(const storage::spi::Result &) override { _gate.countDown(); } void await() { _gate.await(); } @@ -446,7 +457,7 @@ SchemaContext::SchemaContext() : _schema->addSummaryField(Schema::SummaryField("s1", DataType::STRING, CollectionType::SINGLE)); _builder.reset(new DocBuilder(*_schema)); } -SchemaContext::~SchemaContext() {} +SchemaContext::~SchemaContext() = default; struct DocumentContext diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index a6d7f12f199..681d3543ee1 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -23,6 +23,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> @@ -282,8 +283,10 @@ TEST_F("require that we can deserialize old update operations", Fixture) BucketId bucket(toBucket(docId.getGlobalId())); auto upd(f.makeUpdate()); { - UpdateOperation op(UpdateOperation::makeOldUpdate(bucket, Timestamp(10), upd)); - op.serialize(stream); + UpdateOperation op(bucket, Timestamp(10), upd); + op.serializeDocumentOperationOnly(stream); + document::VespaDocumentSerializer serializer(stream); + serializer.write42(*op.getUpdate()); } { UpdateOperation op(FeedOperation::UPDATE_42); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 62c61406f67..a770fff3f5f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -61,6 +61,36 @@ void inject_match_phase_limiting(Properties &setup, const vespalib::string &attr setup.import(cfg); } +FakeResult make_elem_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) { + FakeResult result; + uint32_t pos_should_be_ignored = 0; + for (const auto &doc: match_data) { + result.doc(doc.first); + for (const auto &elem: doc.second) { + result.elem(elem).pos(++pos_should_be_ignored); + } + } + return result; +} + +vespalib::string make_simple_stack_dump(const vespalib::string &field, + const vespalib::string &term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addStringTerm(term, field, 1, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + +vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term, + const vespalib::string &f1_term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addSameElement(2, "ignored field name"); + builder.addStringTerm(a1_term, "a1", 1, search::query::Weight(1)); + builder.addStringTerm(f1_term, "f1", 2, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + //----------------------------------------------------------------------------- const uint32_t NUM_DOCS = 1000; @@ -238,6 +268,13 @@ struct MyWorld { searchContext.attr().addResult("a1", term, result); } + void add_same_element_results(const vespalib::string &a1_term, const vespalib::string &f1_0_term) { + auto a1_result = make_elem_result({{10, {1}}, {20, {2}}, {21, {2}}}); + auto f1_0_result = make_elem_result({{10, {2}}, {20, {2}}, {21, {2}}}); + searchContext.attr().addResult("a1", a1_term, a1_result); + searchContext.idx(0).getFake().addResult("f1", f1_0_term, f1_0_result); + } + void basicResults() { searchContext.idx(0).getFake().addResult("f1", "foo", FakeResult() @@ -249,26 +286,32 @@ struct MyWorld { .doc(600).doc(700).doc(800).doc(900)); } - void setStackDump(Request &request, const vespalib::string &field, - const vespalib::string &term) { - QueryBuilder<ProtonNodeTypes> builder; - builder.addStringTerm(term, field, 1, search::query::Weight(1)); - vespalib::string stack_dump = - StackDumpCreator::create(*builder.build()); + void setStackDump(Request &request, const vespalib::string &stack_dump) { request.stackDump.assign(stack_dump.data(), stack_dump.data() + stack_dump.size()); } - SearchRequest::SP createSimpleRequest(const vespalib::string &field, - const vespalib::string &term) + SearchRequest::SP createRequest(const vespalib::string &stack_dump) { SearchRequest::SP request(new SearchRequest); request->setTimeout(60 * fastos::TimeStamp::SEC); - setStackDump(*request, field, term); + setStackDump(*request, stack_dump); request->maxhits = 10; return request; } + SearchRequest::SP createSimpleRequest(const vespalib::string &field, + const vespalib::string &term) + { + return createRequest(make_simple_stack_dump(field, term)); + } + + SearchRequest::SP createSameElementRequest(const vespalib::string &a1_term, + const vespalib::string &f1_term) + { + return createRequest(make_same_element_stack_dump(a1_term, f1_term)); + } + Matcher::SP createMatcher() { return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, 0); } @@ -317,7 +360,7 @@ struct MyWorld { const vespalib::string & term) { DocsumRequest::SP request(new DocsumRequest); - setStackDump(*request, field, term); + setStackDump(*request, make_simple_stack_dump(field, term)); // match a subset of basic result + request for a non-hit (not // sorted on docid) @@ -800,4 +843,14 @@ TEST("require that fields are tagged with data type") { EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE); } +TEST("require that same element search works (note that this does not test/use the attribute element iterator wrapper)") { + MyWorld world; + world.basicSetup(); + world.add_same_element_results("foo", "bar"); + SearchRequest::SP request = world.createSameElementRequest("foo", "bar"); + SearchReply::UP reply = world.performSearch(request, 1); + ASSERT_EQUAL(1u, reply->hits.size()); + EXPECT_EQUAL(document::DocumentId("doc::20").getGlobalId(), reply->hits[0].gid); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index eb49603f71d..7875e7ec4aa 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for query. -#include <vespa/document/datatype/positiondatatype.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/matchdatareservevisitor.h> #include <vespa/searchcore/proton/matching/blueprintbuilder.h> @@ -26,8 +25,9 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/termasstring.h> +#include <vespa/document/datatype/positiondatatype.h> + #include <vespa/vespalib/testkit/testapp.h> -#include <vector> using document::PositionDataType; using search::fef::FieldInfo; @@ -62,8 +62,7 @@ using std::vector; namespace fef_test = search::fef::test; using CollectionType = FieldInfo::CollectionType; -namespace proton { -namespace matching { +namespace proton::matching { namespace { class Test : public vespalib::TestApp { @@ -175,6 +174,12 @@ Node::UP buildQueryTree(const ViewResolver &resolver, query_builder.addPhrase(2, field, 7, Weight(0)); query_builder.addStringTerm(phrase_term, field, 8, Weight(0)); query_builder.addStringTerm(phrase_term, field, 9, Weight(0)); +#if 0 + //Todo add testing when SameElement blueprints are ready + query_builder.addSameElement(2, field); + query_builder.addStringTerm(string_term, field, 10, Weight(0)); + query_builder.addStringTerm(prefix_term, field, 11, Weight(0)); +#endif Node::UP node = query_builder.build(); ResolveViewVisitor visitor(resolver, idxEnv); @@ -222,19 +227,19 @@ public: EXPECT_EQUAL((double)estimatedHitCount / doc_count, n.field(0).getDocFreq()); } - virtual void visit(ProtonNumberTerm &n) override { checkNode(n, 1, false); } - virtual void visit(ProtonLocationTerm &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonPrefixTerm &n) override { checkNode(n, 1, false); } - virtual void visit(ProtonRangeTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonStringTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonSubstringTerm &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonSuffixTerm &n) override { checkNode(n, 2, false); } - virtual void visit(ProtonPhrase &n) override { checkNode(n, 0, true); } - virtual void visit(ProtonWeightedSetTerm &) override {} - virtual void visit(ProtonDotProduct &) override {} - virtual void visit(ProtonWandTerm &) override {} - virtual void visit(ProtonPredicateQuery &) override {} - virtual void visit(ProtonRegExpTerm &) override {} + void visit(ProtonNumberTerm &n) override { checkNode(n, 1, false); } + void visit(ProtonLocationTerm &n) override { checkNode(n, 0, true); } + void visit(ProtonPrefixTerm &n) override { checkNode(n, 1, false); } + void visit(ProtonRangeTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonStringTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonSubstringTerm &n) override { checkNode(n, 0, true); } + void visit(ProtonSuffixTerm &n) override { checkNode(n, 2, false); } + void visit(ProtonPhrase &n) override { checkNode(n, 0, true); } + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &) override {} }; void Test::requireThatTermsAreLookedUp() { @@ -354,12 +359,12 @@ class SetUpTermDataTestCheckerVisitor int Main() { return 0; } public: - virtual void visit(ProtonNumberTerm &) override {} - virtual void visit(ProtonLocationTerm &) override {} - virtual void visit(ProtonPrefixTerm &) override {} - virtual void visit(ProtonRangeTerm &) override {} + void visit(ProtonNumberTerm &) override {} + void visit(ProtonLocationTerm &) override {} + void visit(ProtonPrefixTerm &) override {} + void visit(ProtonRangeTerm &) override {} - virtual void visit(ProtonStringTerm &n) override { + void visit(ProtonStringTerm &n) override { const ITermData &term_data = n; EXPECT_EQUAL(string_weight.percent(), term_data.getWeight().percent()); @@ -375,17 +380,17 @@ public: } } - virtual void visit(ProtonSubstringTerm &) override {} - virtual void visit(ProtonSuffixTerm &) override {} - virtual void visit(ProtonPhrase &n) override { + void visit(ProtonSubstringTerm &) override {} + void visit(ProtonSuffixTerm &) override {} + void visit(ProtonPhrase &n) override { const ITermData &term_data = n; EXPECT_EQUAL(2u, term_data.getPhraseLength()); } - virtual void visit(ProtonWeightedSetTerm &) override {} - virtual void visit(ProtonDotProduct &) override {} - virtual void visit(ProtonWandTerm &) override {} - virtual void visit(ProtonPredicateQuery &) override {} - virtual void visit(ProtonRegExpTerm &) override {} + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &) override {} }; void Test::requireThatTermDataIsFilledIn() { @@ -855,7 +860,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() EXPECT_EQUAL(exp, act); } -Test::~Test() {} +Test::~Test() = default; int Test::Main() @@ -877,6 +882,7 @@ Test::Main() TEST_CALL(requireThatNearIteratorsCanBeBuilt); TEST_CALL(requireThatONearIteratorsCanBeBuilt); TEST_CALL(requireThatPhraseIteratorsCanBeBuilt); + //TODO Add SameElement testing TEST_CALL(requireThatUnknownFieldActsEmpty); TEST_CALL(requireThatIllegalFieldsAreIgnored); TEST_CALL(requireThatQueryGluesEverythingTogether); @@ -893,7 +899,6 @@ Test::Main() } // namespace -} // namespace matching -} // namespace proton +} // namespace proton::matching TEST_APPHOOK(proton::matching::Test); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index f8a419ba15b..7b6fdd1ae88 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -1,11 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querynodes. -#include <vespa/log/log.h> -LOG_SETUP("querynodes_test"); - #include <vespa/searchcore/proton/matching/querynodes.h> - #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/blueprintbuilder.h> #include <vespa/searchcore/proton/matching/matchdatareservevisitor.h> @@ -33,11 +29,12 @@ LOG_SETUP("querynodes_test"); #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/vespalib/testkit/testapp.h> -#include <cstdarg> -#include <string> -#include <vector> + #include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/log/log.h> +LOG_SETUP("querynodes_test"); + using search::fef::FieldInfo; using search::fef::FieldType; using search::fef::MatchData; @@ -210,9 +207,8 @@ public: }; typedef QueryBuilder<ProtonNodeTypes> QB; -struct Phrase { - void addToBuilder(QB& b) { b.addPhrase(2, view, id, weight); } -}; +struct Phrase { void addToBuilder(QB& b) { b.addPhrase(2, view, id, weight); }}; +struct SameElement { void addToBuilder(QB& b) { b.addSameElement(2, view); }}; struct Near { void addToBuilder(QB& b) { b.addNear(2, distance); } }; struct ONear { void addToBuilder(QB& b) { b.addONear(2, distance); } }; struct Or { void addToBuilder(QB& b) { b.addOr(2); } }; @@ -466,6 +462,11 @@ TEST("requireThatPhrasesGetProperBlending") { TEST_DO(checkProperBlending<Phrase>()); } +TEST("requireThatSameElementGetProperBlending") { + //TODO SameEelement needs proper testing/implementation + //TEST_DO(checkProperBlending<SameElement>()); +} + TEST("requireThatNearGetProperBlending") { TEST_DO(checkProperBlendingWithParent<Near>()); } diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 43d1d43a9d6..2570e64dbe2 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -44,6 +44,7 @@ class Test : public vespalib::TestApp { void requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm(); void requireThatUnrankedTermsAreSkipped(); void requireThatNegativeTermsAreSkipped(); + void requireThatSameElementIsSkipped(); public: int Main() override; @@ -58,6 +59,7 @@ Test::Main() TEST_DO(requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm()); TEST_DO(requireThatUnrankedTermsAreSkipped()); TEST_DO(requireThatNegativeTermsAreSkipped()); + TEST_DO(requireThatSameElementIsSkipped()); TEST_DONE(); } @@ -161,6 +163,24 @@ Test::requireThatNegativeTermsAreSkipped() EXPECT_EQUAL(id[1], term_data[1]->getUniqueId()); } +void +Test::requireThatSameElementIsSkipped() +{ + QueryBuilder<ProtonNodeTypes> query_builder; + query_builder.addAnd(2); + query_builder.addSameElement(2, field); + query_builder.addStringTerm("term1", field, id[0], Weight(1)); + query_builder.addStringTerm("term2", field, id[1], Weight(1)); + query_builder.addStringTerm("term3", field, id[2], Weight(1)); + Node::UP node = query_builder.build(); + + vector<const ITermData *> term_data; + TermDataExtractor::extractTerms(*node, term_data); + EXPECT_EQUAL(1u, term_data.size()); + ASSERT_TRUE(term_data.size() >= 1); + EXPECT_EQUAL(id[2], term_data[0]->getUniqueId()); +} + } // namespace TEST_APPHOOK(Test); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 9262f9a7b6f..705a27c7fc3 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -68,24 +68,8 @@ createUpd(const DocumentType& docType, const DocumentId &docId) return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); } - -document::Document::UP -clone(const document::Document::SP &doc) -{ - return document::Document::UP(doc->clone()); -} - - -document::DocumentUpdate::UP -clone(const document::DocumentUpdate::SP &upd) -{ - return document::DocumentUpdate::UP(upd->clone()); -} - - storage::spi::ClusterState -createClusterState(const storage::lib::State& nodeState = - storage::lib::State::UP) +createClusterState(const storage::lib::State& nodeState = storage::lib::State::UP) { using storage::lib::Distribution; using storage::lib::Node; @@ -99,11 +83,7 @@ createClusterState(const storage::lib::State& nodeState = StorDistributionConfigBuilder dc; cstate.setNodeState(Node(NodeType::STORAGE, 0), - NodeState(NodeType::STORAGE, - nodeState, - "dummy desc", - 1.0, - 1)); + NodeState(NodeType::STORAGE, nodeState, "dummy desc", 1.0, 1)); cstate.setClusterState(State::UP); dc.redundancy = 1; dc.readyCopies = 1; @@ -222,8 +202,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { void handleUpdate(FeedToken token, const Bucket& bucket, Timestamp timestamp, const document::DocumentUpdate::SP& upd) override { - token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), - existingTimestamp > 0); + token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), existingTimestamp > 0); handle(token, bucket, timestamp, upd->getId()); } @@ -312,8 +291,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { return frozen.find(bucket.getBucketId().getId()) != frozen.end(); } bool wasFrozen(const Bucket &bucket) { - return was_frozen.find(bucket.getBucketId().getId()) - != was_frozen.end(); + return was_frozen.find(bucket.getBucketId().getId()) != was_frozen.end(); } }; @@ -335,7 +313,7 @@ HandlerSet::HandlerSet() handler1(static_cast<MyHandler &>(*phandler1.get())), handler2(static_cast<MyHandler &>(*phandler2.get())) {} -HandlerSet::~HandlerSet() {} +HandlerSet::~HandlerSet() = default; DocumentType type1(createDocType("type1", 1)); DocumentType type2(createDocType("type2", 2)); @@ -405,8 +383,8 @@ struct SimpleResourceWriteFilter : public IResourceWriteFilter _message() {} - virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; } - virtual State getAcceptState() const override { + bool acceptWriteOperation() const override { return _acceptWriteOperation; } + State getAcceptState() const override { return IResourceWriteFilter::State(acceptWriteOperation(), _message); } }; @@ -475,8 +453,7 @@ TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture) TEST_F("require that puts are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.put(bucket1, tstamp1, doc1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -485,20 +462,16 @@ TEST_F("require that puts are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.put(bucket1, tstamp1, doc3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.put(bucket1, tstamp1, doc3, context)); } TEST_F("require that puts with old id scheme are rejected", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "Old id scheme not supported in " - "elastic mode (doc:old:id-scheme)"), - f.engine.put(bucket1, tstamp1, old_doc, context)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "Old id scheme not supported in elastic mode (doc:old:id-scheme)"), + f.engine.put(bucket1, tstamp1, old_doc, context)); } @@ -508,8 +481,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, "Put operation rejected for document 'doc:old:id-scheme': 'Disk is full'"), @@ -520,8 +492,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur TEST_F("require that updates are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.setExistingTimestamp(tstamp2); UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); @@ -534,9 +505,8 @@ TEST_F("require that updates are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); EXPECT_EQUAL(tstamp3, ur.getExistingTimestamp()); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.update(bucket1, tstamp1, upd3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.update(bucket1, tstamp1, upd3, context)); } @@ -546,8 +516,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, @@ -559,8 +528,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix TEST_F("require that removes are routed to handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context); assertHandler(bucket0, tstamp0, docId0, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -598,8 +566,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", Simpl f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context)); } @@ -628,8 +595,7 @@ TEST_F("require that setActiveState() is routed to handlers and merged", SimpleF f.hset.handler1.bucketStateResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.bucketStateResult = Result(Result::PERMANENT_ERROR, "err2"); - Result result = f.engine.setActiveState(bucket1, - storage::spi::BucketInfo::NOT_ACTIVE); + Result result = f.engine.setActiveState(bucket1, storage::spi::BucketInfo::NOT_ACTIVE); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); EXPECT_EQUAL("err1, err2", result.getErrorMessage()); EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler1.lastBucketState); @@ -651,16 +617,12 @@ TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFi } -TEST_F("require that createBucket() is routed to handlers and merged", - SimpleFixture) +TEST_F("require that createBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - f.hset.handler1._createBucketResult = - Result(Result::TRANSIENT_ERROR, "err1a"); - f.hset.handler2._createBucketResult = - Result(Result::PERMANENT_ERROR, "err2a"); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + f.hset.handler1._createBucketResult = Result(Result::TRANSIENT_ERROR, "err1a"); + f.hset.handler2._createBucketResult = Result(Result::PERMANENT_ERROR, "err2a"); Result result = f.engine.createBucket(bucket1, context); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); @@ -671,8 +633,7 @@ TEST_F("require that createBucket() is routed to handlers and merged", TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.deleteBucketResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.deleteBucketResult = Result(Result::PERMANENT_ERROR, "err2"); @@ -691,10 +652,8 @@ TEST_F("require that getModifiedBuckets() is routed to handlers and merged", Sim TEST_F("require that get is sent to all handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(docId1, f.hset.handler2.lastDocId); @@ -704,8 +663,7 @@ TEST_F("require that get freezes the bucket", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1)); @@ -717,10 +675,8 @@ TEST_F("require that get returns the first document found", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(DocumentId(), f.hset.handler2.lastDocId); @@ -732,8 +688,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) { TEST_F("require that createIterator does", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -741,15 +696,13 @@ TEST_F("require that createIterator does", SimpleFixture) { EXPECT_TRUE(result.getIteratorId()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); } TEST_F("require that iterator ids are unique", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -764,10 +717,8 @@ TEST_F("require that iterator ids are unique", SimpleFixture) { TEST_F("require that iterate requires valid iterator", SimpleFixture) { uint64_t max_size = 1024; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage()); @@ -786,16 +737,14 @@ TEST_F("require that iterate returns documents", SimpleFixture) { f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); uint64_t max_size = 1024; CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(result.getIteratorId()); - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); EXPECT_EQUAL(2u, it_result.getEntries().size()); } @@ -804,33 +753,28 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(create_result.getIteratorId()); - Result result = f.engine.destroyIterator(create_result.getIteratorId(), - context); + Result result = f.engine.destroyIterator(create_result.getIteratorId(), context); EXPECT_FALSE(result.hasError()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(create_result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(create_result.getIteratorId(), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); string msg_prefix = "Unknown iterator with id"; - EXPECT_EQUAL(msg_prefix, - it_result.getErrorMessage().substr(0, msg_prefix.size())); + EXPECT_EQUAL(msg_prefix, it_result.getErrorMessage().substr(0, msg_prefix.size())); } TEST_F("require that buckets are frozen during iterator life", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 5b6a9faa05d..30f6c2d92c2 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -23,6 +23,7 @@ vespa_add_library(searchcore_attribute STATIC attributedisklayout.cpp attributemanager.cpp attributesconfigscout.cpp + document_field_extractor.cpp document_field_populator.cpp document_field_retriever.cpp exclusive_attribute_read_accessor.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index a6329db2aee..35f5f09fc37 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_writer.h" +#include "ifieldupdatecallback.h" #include "attributemanager.h" +#include "document_field_extractor.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -21,10 +23,34 @@ namespace proton { using LidVector = LidVectorContext::LidVector; +AttributeWriter::WriteField::WriteField(AttributeVector &attribute) + : _fieldPath(), + _attribute(attribute), + _compoundAttribute(false) +{ + const vespalib::string &name = attribute.getName(); + _compoundAttribute = name.find('.') != vespalib::string::npos; +} + +AttributeWriter::WriteField::~WriteField() = default; + +void +AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) +{ + const vespalib::string &name = _attribute.getName(); + FieldPath fp; + try { + docType.buildFieldPath(fp, name); + } catch (document::FieldNotFoundException & e) { + fp = FieldPath(); + } + _fieldPath = std::move(fp); +} + AttributeWriter::WriteContext::WriteContext(uint32_t executorId) : _executorId(executorId), - _fieldPaths(), - _attributes() + _fields(), + _hasCompoundAttribute(false) { } @@ -36,28 +62,20 @@ AttributeWriter::WriteContext::~WriteContext() = default; AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) = default; void -AttributeWriter::WriteContext::add(AttributeVector *attr) +AttributeWriter::WriteContext::add(AttributeVector &attr) { - _attributes.emplace_back(attr); - _fieldPaths.emplace_back(); + _fields.emplace_back(attr); + if (_fields.back().getCompoundAttribute()) { + _hasCompoundAttribute = true; + } } void AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType) { - size_t fieldId = 0; - for (const auto &attrp : _attributes) { - const vespalib::string &name = attrp->getName(); - FieldPath fp; - try { - docType.buildFieldPath(fp, name); - } catch (document::FieldNotFoundException & e) { } - - assert(fieldId < _fieldPaths.size()); - _fieldPaths[fieldId] = std::move(fp); - ++fieldId; + for (auto &field : _fields) { + field.buildFieldPath(docType); } - assert(fieldId == _fieldPaths.size()); } namespace { @@ -197,29 +215,30 @@ class PutTask : public vespalib::Executor::Task const SerialNum _serialNum; const uint32_t _lid; const bool _immediateCommit; + const bool _allAttributes; std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone; std::vector<FieldValue::UP> _fieldValues; public: - PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, const Document &doc, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone); + PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone); virtual ~PutTask() override; virtual void run() override; }; -PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, const Document &doc, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone) +PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), _lid(lid), _immediateCommit(immediateCommit), + _allAttributes(allAttributes), _onWriteDone(onWriteDone) { - const auto &fieldPaths = _wc.getFieldPaths(); - _fieldValues.reserve(fieldPaths.size()); - for (const auto &fieldPath : fieldPaths) { - FieldValue::UP fv; - if (!fieldPath.empty()) { - fv = doc.getNestedFieldValue(fieldPath.getFullRange()); + const auto &fields = _wc.getFields(); + _fieldValues.reserve(fields.size()); + for (const auto &field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + FieldValue::UP fv = fieldExtractor.getFieldValue(field.getFieldPath()); + _fieldValues.emplace_back(std::move(fv)); } - _fieldValues.emplace_back(std::move(fv)); } } @@ -231,13 +250,15 @@ void PutTask::run() { uint32_t fieldId = 0; - const auto &attributes = _wc.getAttributes(); - for (auto attrp : attributes) { - AttributeVector &attr = *attrp; - if (attr.getStatus().getLastSyncToken() < _serialNum) { - applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + const auto &fields = _wc.getFields(); + for (auto field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + AttributeVector &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { + applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + } + ++fieldId; } - ++fieldId; } } @@ -271,9 +292,9 @@ RemoveTask::~RemoveTask() void RemoveTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); // Must use <= due to how move operations are handled if (attr.getStatus().getLastSyncToken() <= _serialNum) { applyRemoveToAttribute(_serialNum, _lid, _immediateCommit, attr, _onWriteDone); @@ -303,13 +324,14 @@ public: {} virtual ~BatchRemoveTask() override {} virtual void run() override { - for (auto attr : _writeCtx.getAttributes()) { - if (attr->getStatus().getLastSyncToken() < _serialNum) { + for (auto field : _writeCtx.getFields()) { + auto &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { for (auto lidToRemove : _lidsToRemove) { - applyRemoveToAttribute(_serialNum, lidToRemove, false, *attr, _onWriteDone); + applyRemoveToAttribute(_serialNum, lidToRemove, false, attr, _onWriteDone); } if (_immediateCommit) { - attr->commit(_serialNum, _serialNum); + attr.commit(_serialNum, _serialNum); } } } @@ -342,9 +364,9 @@ CommitTask::~CommitTask() void CommitTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); applyCommit(_serialNum, _onWriteDone, attr); } } @@ -365,7 +387,12 @@ AttributeWriter::setupWriteContexts() (_writeContexts.back().getExecutorId() != fc.getExecutorId())) { _writeContexts.emplace_back(fc.getExecutorId()); } - _writeContexts.back().add(fc.getAttribute()); + _writeContexts.back().add(*fc.getAttribute()); + } + for (const auto &wc : _writeContexts) { + if (wc.getHasCompoundAttribute()) { + _hasCompoundAttribute = true; + } } } @@ -380,11 +407,18 @@ AttributeWriter::buildFieldPaths(const DocumentType & docType, const DataType *d void AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone) { + const DataType *dataType(doc.getDataType()); + if (_dataType != dataType) { + buildFieldPaths(doc.getType(), dataType); + } + DocumentFieldExtractor extractor(doc); for (const auto &wc : _writeContexts) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, doc, lid, immediateCommit, onWriteDone); - _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + if (allAttributes || wc.getHasCompoundAttribute()) { + auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + } } } @@ -404,7 +438,8 @@ AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) _attributeFieldWriter(mgr->getAttributeFieldWriter()), _writableAttributes(mgr->getWritableAttributes()), _writeContexts(), - _dataType(nullptr) + _dataType(nullptr), + _hasCompoundAttribute(false) { setupWriteContexts(); } @@ -431,18 +466,26 @@ void AttributeWriter::put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) { - FieldValue::UP attrVal; LOG(spam, "Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", serialNum, doc.getId().toString().c_str(), lid, doc.toString(true).c_str()); - const DataType *dataType(doc.getDataType()); - if (_dataType != dataType) { - buildFieldPaths(doc.getType(), dataType); - } - internalPut(serialNum, doc, lid, immediateCommit, onWriteDone); + internalPut(serialNum, doc, lid, immediateCommit, true, onWriteDone); +} + +void +AttributeWriter::update(SerialNum serialNum, const Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone) +{ + LOG(spam, + "Handle update: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", + serialNum, + doc.getId().toString().c_str(), + lid, + doc.toString(true).c_str()); + internalPut(serialNum, doc, lid, immediateCommit, false, onWriteDone); } void @@ -464,17 +507,15 @@ AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum, void AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) { LOG(debug, "Inspecting update for document %d.", lid); for (const auto &fupd : upd.getUpdates()) { - LOG(debug, "Retrieving guard for attribute vector '%s'.", - fupd.getField().getName().c_str()); - AttributeVector *attrp = - _mgr->getWritableAttribute(fupd.getField().getName()); + LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); + AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName()); + onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { - LOG(spam, "Failed to find attribute vector %s", - fupd.getField().getName().c_str()); + LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); continue; } AttributeVector &attr = *attrp; @@ -483,8 +524,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document if (attr.getStatus().getLastSyncToken() >= serialNum) continue; - LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", - lid, attr.getName().c_str()); + LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attr.getName().c_str()); // NOTE: The lifetime of the field update will be ensured by keeping the document update alive // in a operation done context object. @@ -549,5 +589,11 @@ AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) _attributeFieldWriter.sync(); } +bool +AttributeWriter::getHasCompoundAttribute() const +{ + return _hasCompoundAttribute; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index cbda11180c0..dfdafa3bea9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -23,30 +23,44 @@ private: search::ISequencedTaskExecutor &_attributeFieldWriter; const std::vector<search::AttributeVector *> &_writableAttributes; public: + class WriteField + { + FieldPath _fieldPath; + AttributeVector &_attribute; + bool _compoundAttribute; // in array/map of struct + public: + WriteField(AttributeVector &attribute); + ~WriteField(); + AttributeVector &getAttribute() const { return _attribute; } + const FieldPath &getFieldPath() const { return _fieldPath; } + void buildFieldPath(const DocumentType &docType); + bool getCompoundAttribute() const { return _compoundAttribute; } + }; class WriteContext { uint32_t _executorId; - std::vector<FieldPath> _fieldPaths; - std::vector<AttributeVector *> _attributes; + std::vector<WriteField> _fields; + bool _hasCompoundAttribute; public: WriteContext(uint32_t executorId); WriteContext(WriteContext &&rhs); ~WriteContext(); WriteContext &operator=(WriteContext &&rhs); void buildFieldPaths(const DocumentType &docType); - void add(AttributeVector *attr); + void add(AttributeVector &attr); uint32_t getExecutorId() const { return _executorId; } - const std::vector<FieldPath> &getFieldPaths() const { return _fieldPaths; } - const std::vector<AttributeVector *> &getAttributes() const { return _attributes; } + const std::vector<WriteField> &getFields() const { return _fields; } + bool getHasCompoundAttribute() const { return _hasCompoundAttribute; } }; private: std::vector<WriteContext> _writeContexts; const DataType *_dataType; + bool _hasCompoundAttribute; void setupWriteContexts(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone); + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone); void internalRemove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone); @@ -68,6 +82,8 @@ public: void remove(const LidVector &lidVector, SerialNum serialNum, bool immediateCommit, OnWriteDoneType onWriteDone) override; void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) override; void heartBeat(SerialNum serialNum) override; void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override; @@ -76,7 +92,8 @@ public: } void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; - virtual void onReplayDone(uint32_t docIdLimit) override; + void onReplayDone(uint32_t docIdLimit) override; + bool getHasCompoundAttribute() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp new file mode 100644 index 00000000000..46f3fdeff67 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -0,0 +1,243 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_field_extractor.h" +#include <vespa/document/datatype/arraydatatype.h> +#include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/bytefieldvalue.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/doublefieldvalue.h> +#include <vespa/document/fieldvalue/floatfieldvalue.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/longfieldvalue.h> +#include <vespa/document/fieldvalue/shortfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/fieldvalue/structfieldvalue.h> +#include <vespa/document/fieldvalue/mapfieldvalue.h> +#include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> + +using document::FieldValue; +using document::ByteFieldValue; +using document::ShortFieldValue; +using document::IntFieldValue; +using document::LongFieldValue; +using document::FloatFieldValue; +using document::DoubleFieldValue; +using document::StringFieldValue; +using document::StructFieldValue; +using document::MapFieldValue; +using document::DataType; +using document::ArrayDataType; +using document::ArrayFieldValue; +using document::Document; +using document::FieldPath; +using document::FieldPathEntry; +using document::FieldValueVisitor; +using vespalib::IllegalStateException; +using vespalib::make_string; +using search::attribute::getUndefined; + +namespace proton { + +namespace { + +class SetUndefinedValueVisitor : public FieldValueVisitor +{ + void visit(document::AnnotationReferenceFieldValue &) override { } + void visit(ArrayFieldValue &) override { } + void visit(ByteFieldValue &value) override { value = getUndefined<int8_t>(); } + void visit(Document &) override { } + void visit(DoubleFieldValue &value) override { value = getUndefined<double>(); } + void visit(FloatFieldValue &value) override { value = getUndefined<float>(); } + void visit(IntFieldValue &value) override { value = getUndefined<int32_t>(); } + void visit(LongFieldValue &value) override { value = getUndefined<int64_t>(); } + void visit(MapFieldValue &) override { } + void visit(document::PredicateFieldValue &) override { } + void visit(document::RawFieldValue &) override { } + void visit(ShortFieldValue &value) override { value = getUndefined<int16_t>(); } + void visit(StringFieldValue &) override { } + void visit(StructFieldValue &) override { } + void visit(document::WeightedSetFieldValue &) override { } + void visit(document::TensorFieldValue &) override { } + void visit(document::ReferenceFieldValue &) override { } +}; + +SetUndefinedValueVisitor setUndefinedValueVisitor; + +const ArrayDataType arrayTypeByte(*DataType::BYTE); +const ArrayDataType arrayTypeShort(*DataType::SHORT); +const ArrayDataType arrayTypeInt(*DataType::INT); +const ArrayDataType arrayTypeLong(*DataType::LONG); +const ArrayDataType arrayTypeFloat(*DataType::FLOAT); +const ArrayDataType arrayTypeDouble(*DataType::DOUBLE); +const ArrayDataType arrayTypeString(*DataType::STRING); + +const DataType * +getArrayType(const DataType &fieldType) +{ + switch (fieldType.getId()) { + case DataType::Type::T_BYTE: + return &arrayTypeByte; + case DataType::Type::T_SHORT: + return &arrayTypeShort; + case DataType::Type::T_INT: + return &arrayTypeInt; + case DataType::Type::T_LONG: + return &arrayTypeLong; + case DataType::Type::T_FLOAT: + return &arrayTypeFloat; + case DataType::Type::T_DOUBLE: + return &arrayTypeDouble; + case DataType::Type::T_STRING: + return &arrayTypeString; + default: + return nullptr; + } +} + +std::unique_ptr<ArrayFieldValue> +makeArray(const FieldPathEntry &fieldPathEntry, size_t size) +{ + const auto arrayType = getArrayType(fieldPathEntry.getDataType()); + auto array = std::make_unique<ArrayFieldValue>(*arrayType); + array->resize(size); + return array; +} + +bool +checkInherits(const FieldValue &fieldValue, unsigned id) +{ + const vespalib::Identifiable::RuntimeClass &rc = fieldValue.getClass(); + return rc.inherits(id); +} + +} + +DocumentFieldExtractor::DocumentFieldExtractor(const Document &doc) + : _doc(doc), + _cachedFieldValues() +{ +} + +DocumentFieldExtractor::~DocumentFieldExtractor() = default; + +bool +DocumentFieldExtractor::isSupported(const FieldPath &fieldPath) +{ + if (!fieldPath.empty() && + fieldPath[0].getType() != FieldPathEntry::Type::STRUCT_FIELD) { + return false; + } + if (fieldPath.size() == 2) { + if (fieldPath[1].getType() != FieldPathEntry::Type::STRUCT_FIELD && + fieldPath[1].getType() != FieldPathEntry::Type::MAP_ALL_KEYS) { + return false; + } + } else if (fieldPath.size() == 3) { + if (fieldPath[1].getType() != FieldPathEntry::Type::MAP_ALL_VALUES || + fieldPath[2].getType() != FieldPathEntry::Type::STRUCT_FIELD) { + return false; + } + } else if (fieldPath.size() > 3) { + return false; + } + return true; +} + +const FieldValue * +DocumentFieldExtractor::getCachedFieldValue(const FieldPathEntry &fieldPathEntry) +{ + auto itr = _cachedFieldValues.find(fieldPathEntry.getName()); + if (itr != _cachedFieldValues.end()) { + return itr->second.get(); + } else { + auto insres = _cachedFieldValues.insert(std::make_pair(fieldPathEntry.getName(), _doc.getValue(fieldPathEntry.getFieldRef()))); + assert(insres.second); + return insres.first->second.get(); + } +} + +std::unique_ptr<FieldValue> +DocumentFieldExtractor::getSimpleFieldValue(const FieldPath &fieldPath) +{ + return _doc.getNestedFieldValue(fieldPath.getFullRange()); +} + +std::unique_ptr<FieldValue> +DocumentFieldExtractor::getStructArrayFieldValue(const FieldPath &fieldPath) +{ + const auto outerFieldValue = getCachedFieldValue(fieldPath[0]); + if (outerFieldValue != nullptr && checkInherits(*outerFieldValue, ArrayFieldValue::classId)) { + const auto outerArray = static_cast<const ArrayFieldValue *>(outerFieldValue); + const auto &innerFieldPathEntry = fieldPath[1]; + auto array = makeArray(innerFieldPathEntry, outerArray->size()); + uint32_t arrayIndex = 0; + for (const auto &outerElemBase : *outerArray) { + auto &arrayElem = (*array)[arrayIndex++]; + const auto &structElem = static_cast<const StructFieldValue &>(outerElemBase); + if (!structElem.getValue(innerFieldPathEntry.getFieldRef(), arrayElem)) { + arrayElem.accept(setUndefinedValueVisitor); + } + } + return array; + } + return std::unique_ptr<FieldValue>(); +} + +std::unique_ptr<FieldValue> +DocumentFieldExtractor::getStructMapKeyFieldValue(const FieldPath &fieldPath) +{ + const auto outerFieldValue = getCachedFieldValue(fieldPath[0]); + if (outerFieldValue != nullptr && checkInherits(*outerFieldValue, MapFieldValue::classId)) { + const auto outerMap = static_cast<const MapFieldValue *>(outerFieldValue); + auto array = makeArray(fieldPath[1], outerMap->size()); + uint32_t arrayIndex = 0; + for (const auto &mapElem : *outerMap) { + (*array)[arrayIndex++].assign(*mapElem.first); + } + return array; + } + return std::unique_ptr<FieldValue>(); +} + +std::unique_ptr<document::FieldValue> +DocumentFieldExtractor::getStructMapFieldValue(const FieldPath &fieldPath) +{ + const auto outerFieldValue = getCachedFieldValue(fieldPath[0]); + if (outerFieldValue != nullptr && checkInherits(*outerFieldValue, MapFieldValue::classId)) { + const auto outerMap = static_cast<const MapFieldValue *>(outerFieldValue); + const auto &innerFieldPathEntry = fieldPath[2]; + auto array = makeArray(innerFieldPathEntry, outerMap->size()); + uint32_t arrayIndex = 0; + for (const auto &mapElem : *outerMap) { + auto &arrayElem = (*array)[arrayIndex++]; + const auto &structElem = static_cast<const StructFieldValue &>(*mapElem.second); + if (!structElem.getValue(innerFieldPathEntry.getFieldRef(), arrayElem)) { + arrayElem.accept(setUndefinedValueVisitor); + } + } + return array; + } + return std::unique_ptr<FieldValue>(); +} + +std::unique_ptr<FieldValue> +DocumentFieldExtractor::getFieldValue(const FieldPath &fieldPath) +{ + if (fieldPath.size() == 1) { + return getSimpleFieldValue(fieldPath); + } else if (fieldPath.size() == 2) { + if (fieldPath[1].getType() == FieldPathEntry::Type::STRUCT_FIELD) { + return getStructArrayFieldValue(fieldPath); + } else { + return getStructMapKeyFieldValue(fieldPath); + } + } else if (fieldPath.size() == 3) { + return getStructMapFieldValue(fieldPath); + } + return std::unique_ptr<FieldValue>(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h new file mode 100644 index 00000000000..48e2da9c4c6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h @@ -0,0 +1,46 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <memory> + +namespace document +{ + +class Document; +class FieldValue; +class FieldPath; +class FieldPathEntry; + +} + +namespace proton { + +/** + * Class used to extract a field value from a document field or from a + * nested field in an array/map of structs. + */ +class DocumentFieldExtractor +{ + const document::Document &_doc; + vespalib::hash_map<vespalib::string, std::unique_ptr<document::FieldValue>> _cachedFieldValues; + + const document::FieldValue *getCachedFieldValue(const document::FieldPathEntry &fieldPathEntry); + std::unique_ptr<document::FieldValue> getSimpleFieldValue(const document::FieldPath &fieldPath); + std::unique_ptr<document::FieldValue> getStructArrayFieldValue(const document::FieldPath &fieldPath); + std::unique_ptr<document::FieldValue> getStructMapKeyFieldValue(const document::FieldPath &fieldPath); + std::unique_ptr<document::FieldValue> getStructMapFieldValue(const document::FieldPath &fieldPath); + +public: + DocumentFieldExtractor(const document::Document &doc); + ~DocumentFieldExtractor(); + + std::unique_ptr<document::FieldValue> getFieldValue(const document::FieldPath &fieldPath); + + /** + * Check if fieldPath is in a supported form. + */ + static bool isSupported(const document::FieldPath &fieldPath); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h index abcf132d537..7a557b17964 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h @@ -1,18 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> +#include "i_attribute_manager.h" +#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/query/base.h> #include <vespa/searchlib/common/serialnum.h> -#include <vespa/searchcore/proton/attribute/i_attribute_manager.h> -#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> namespace search { class IDestructorCallback; } namespace proton { +class IFieldUpdateCallback; + /** * Interface for an attribute writer that handles writes in form of put, update and remove * to an underlying set of attribute vectors. @@ -31,10 +33,8 @@ public: virtual ~IAttributeWriter() {} - virtual std::vector<search::AttributeVector *> - getWritableAttributes() const = 0; - virtual search::AttributeVector * - getWritableAttribute(const vespalib::string &attrName) const = 0; + virtual std::vector<search::AttributeVector *> getWritableAttributes() const = 0; + virtual search::AttributeVector *getWritableAttribute(const vespalib::string &attrName) const = 0; virtual void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, @@ -46,6 +46,11 @@ public: * The OnWriteDoneType instance should ensure the lifetime of the given DocumentUpdate instance. */ virtual void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) = 0; + /* + * Update the underlying compound attributes based on updated document. + */ + virtual void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(SerialNum serialNum) = 0; /** @@ -60,6 +65,8 @@ public: virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void onReplayDone(uint32_t docIdLimit) = 0; + + virtual bool getHasCompoundAttribute() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h new file mode 100644 index 00000000000..ffb8555cd2c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search { class AttributeVector; } + +namespace proton { + +struct IFieldUpdateCallback { + virtual ~IFieldUpdateCallback() { } + virtual void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) = 0; +}; + +struct DummyFieldUpdateCallback : IFieldUpdateCallback { + void onUpdateField(vespalib::stringref, const search::AttributeVector *) override {} +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index 068b7f4fa00..e10e6ed539b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -12,15 +12,14 @@ using search::QueryTermSimple; using search::fef::TermFieldMatchData; using search::queryeval::SearchIterator; -namespace proton { -namespace documentmetastore { +namespace proton::documentmetastore { namespace { class GidAllSearchIterator : public AttributeIteratorBase { private: - virtual void + void doSeek(uint32_t docId) override { if (_store.validLidFast(docId)) { @@ -28,7 +27,7 @@ private: } } - virtual void + void doUnpack(uint32_t docId) override { _matchData->reset(docId); @@ -37,8 +36,7 @@ private: protected: const DocumentMetaStore & _store; public: - GidAllSearchIterator(TermFieldMatchData *matchData, - const DocumentMetaStore &store) + GidAllSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store) : AttributeIteratorBase(matchData), _store(store) { @@ -79,7 +77,7 @@ class GidSearchIterator : public GidAllSearchIterator private: const GlobalId & _gid; - virtual void + void doSeek(uint32_t docId) override { AttributeVector::DocId lid = 0; @@ -90,9 +88,7 @@ private: } } public: - GidSearchIterator(TermFieldMatchData *matchData, - const DocumentMetaStore &store, - const GlobalId &gid) + GidSearchIterator(TermFieldMatchData *matchData, const DocumentMetaStore &store, const GlobalId &gid) : GidAllSearchIterator(matchData, store), _gid(gid) { @@ -101,23 +97,16 @@ public: } -bool -SearchContext::onCmp(DocId docId, int32_t &weight) const +int32_t +SearchContext::onFind(DocId, int32_t, int32_t &) const { - (void) docId; - (void) weight; - throw vespalib::IllegalStateException( - "The function is not implemented for documentmetastore::SearchContext"); - return false; + throw vespalib::IllegalStateException("The function is not implemented for documentmetastore::SearchContext"); } -bool -SearchContext::onCmp(DocId docId) const +int32_t +SearchContext::onFind(DocId, int32_t ) const { - (void) docId; - throw vespalib::IllegalStateException( - "The function is not implemented for documentmetastore::SearchContext"); - return false; + throw vespalib::IllegalStateException("The function is not implemented for documentmetastore::SearchContext"); } unsigned int @@ -127,15 +116,13 @@ SearchContext::approximateHits() const } SearchIterator::UP -SearchContext::createIterator(TermFieldMatchData *matchData, - bool strict) +SearchContext::createIterator(TermFieldMatchData *matchData, bool strict) { return _isWord - ? SearchIterator::UP(new GidSearchIterator(matchData, getStore(), _gid)) + ? std::make_unique<GidSearchIterator>(matchData, getStore(), _gid) : strict - ? SearchIterator::UP(new GidStrictAllSearchIterator(matchData, - getStore())) - : SearchIterator::UP(new GidAllSearchIterator(matchData, getStore())); + ? std::make_unique<GidStrictAllSearchIterator>(matchData, getStore()) + : std::make_unique<GidAllSearchIterator>(matchData, getStore()); } const DocumentMetaStore & @@ -144,12 +131,10 @@ SearchContext::getStore() const return static_cast<const DocumentMetaStore &>(attribute()); } -SearchContext::SearchContext(QueryTermSimple::UP qTerm, - const DocumentMetaStore &toBeSearched) +SearchContext::SearchContext(QueryTermSimple::UP qTerm, const DocumentMetaStore &toBeSearched) : search::AttributeVector::SearchContext(toBeSearched), _isWord(qTerm->isWord()) { } } -} diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h index afd09be43e9..31c520bedf5 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include "documentmetastore.h" -namespace proton { -namespace documentmetastore { +namespace proton::documentmetastore { /** * Search context used to search the document meta store for all valid documents. @@ -15,14 +14,14 @@ namespace documentmetastore { class SearchContext : public search::AttributeVector::SearchContext { private: - typedef search::AttributeVector::DocId DocId; + using DocId = search::AttributeVector::DocId; bool _isWord; document::GlobalId _gid; unsigned int approximateHits() const override; - bool onCmp(DocId docId, int32_t &weight) const override; - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; + int32_t onFind(DocId docId, int32_t elemId) const override; search::queryeval::SearchIterator::UP createIterator(search::fef::TermFieldMatchData *matchData, bool strict) override; @@ -30,10 +29,7 @@ private: const DocumentMetaStore &getStore() const; public: - SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, - const DocumentMetaStore &toBeSearched); + SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, const DocumentMetaStore &toBeSearched); }; -} // namespace documentmetastore -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index b2567527560..37b23449315 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -26,9 +26,7 @@ DocumentOperation::DocumentOperation(Type type) } -DocumentOperation::DocumentOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp) +DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const Timestamp ×tamp) : FeedOperation(type), _bucketId(bucketId), _timestamp(timestamp), @@ -62,7 +60,12 @@ vespalib::string DocumentOperation::docArgsToString() const { } void -DocumentOperation::serialize(vespalib::nbostream &os) const +DocumentOperation::serialize(vespalib::nbostream &os) const { + serializeDocumentOperationOnly(os); +} + +void +DocumentOperation::serializeDocumentOperationOnly(vespalib::nbostream &os) const { os << _bucketId; os << _timestamp; @@ -74,8 +77,7 @@ DocumentOperation::serialize(vespalib::nbostream &os) const void -DocumentOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &) +DocumentOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &) { is >> _bucketId; is >> _timestamp; @@ -85,4 +87,8 @@ DocumentOperation::deserialize(vespalib::nbostream &is, is >> _prevTimestamp; } + DbDocumentId DocumentOperation::getDbDocumentId() const { + return _dbdId; + } + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index bc961c580fc..9a823c553bd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -22,124 +22,35 @@ protected: DocumentOperation(Type type); - DocumentOperation(Type type, - const document::BucketId &bucketId, + DocumentOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp); void assertValidBucketId(const document::DocumentId &docId) const; vespalib::string docArgsToString() const; public: - virtual - ~DocumentOperation() - { - } - - const - document::BucketId & - getBucketId() const - { - return _bucketId; - } - - storage::spi::Timestamp - getTimestamp() const - { - return _timestamp; - } - - search::DocumentIdT - getLid() const - { - return _dbdId.getLid(); - } - - search::DocumentIdT - getPrevLid() const - { - return _prevDbdId.getLid(); - } - - uint32_t - getSubDbId() const - { - return _dbdId.getSubDbId(); - } - - uint32_t - getPrevSubDbId() const - { - return _prevDbdId.getSubDbId(); - } - - bool - getValidDbdId() const - { - return _dbdId.valid(); - } - - bool - getValidDbdId(uint32_t subDbId) const - { - return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; - } - - bool - getValidPrevDbdId() const - { - return _prevDbdId.valid(); - } - - bool - getValidPrevDbdId(uint32_t subDbId) const - { - return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; - } - - bool - changedDbdId() const - { - return _dbdId != _prevDbdId; - } - bool - getPrevMarkedAsRemoved() const - { - return _prevMarkedAsRemoved; - } - - void - setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) - { - _prevMarkedAsRemoved = prevMarkedAsRemoved; - } - - DbDocumentId - getDbDocumentId() const - { - return _dbdId; - } - - DbDocumentId - getPrevDbDocumentId() const - { - return _prevDbdId; - } - - void - setDbDocumentId(DbDocumentId dbdId) - { - _dbdId = dbdId; - } - - void - setPrevDbDocumentId(DbDocumentId prevDbdId) - { - _prevDbdId = prevDbdId; - } - - search::DocumentIdT - getNewOrPrevLid(uint32_t subDbId) const - { + ~DocumentOperation() override {} + const document::BucketId &getBucketId() const { return _bucketId; } + storage::spi::Timestamp getTimestamp() const { return _timestamp; } + + search::DocumentIdT getLid() const { return _dbdId.getLid(); } + search::DocumentIdT getPrevLid() const { return _prevDbdId.getLid(); } + uint32_t getSubDbId() const { return _dbdId.getSubDbId(); } + uint32_t getPrevSubDbId() const { return _prevDbdId.getSubDbId(); } + bool getValidDbdId() const { return _dbdId.valid(); } + bool getValidDbdId(uint32_t subDbId) const { return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; } + bool getValidPrevDbdId() const { return _prevDbdId.valid(); } + bool getValidPrevDbdId(uint32_t subDbId) const { return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; } + bool changedDbdId() const { return _dbdId != _prevDbdId; } + bool getPrevMarkedAsRemoved() const { return _prevMarkedAsRemoved; } + void setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) { _prevMarkedAsRemoved = prevMarkedAsRemoved; } + DbDocumentId getDbDocumentId() const; + DbDocumentId getPrevDbDocumentId() const { return _prevDbdId; } + + void setDbDocumentId(DbDocumentId dbdId) { _dbdId = dbdId; } + void setPrevDbDocumentId(DbDocumentId prevDbdId) { _prevDbdId = prevDbdId; } + + search::DocumentIdT getNewOrPrevLid(uint32_t subDbId) const { if (getValidDbdId() && getSubDbId() == subDbId) return getLid(); if (getValidPrevDbdId() && getPrevSubDbId() == subDbId) @@ -147,51 +58,34 @@ public: return 0; } - bool - getValidNewOrPrevDbdId() const - { + bool getValidNewOrPrevDbdId() const { return getValidDbdId() || getValidPrevDbdId(); } - bool - notMovingLidInSameSubDb() const - { + bool notMovingLidInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() == getPrevLid(); } - bool - movingLidIfInSameSubDb() const - { + bool movingLidIfInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() != getPrevLid(); } - storage::spi::Timestamp - getPrevTimestamp() const - { - return _prevTimestamp; - } - - void - setPrevTimestamp(storage::spi::Timestamp prevTimestamp) - { - _prevTimestamp = prevTimestamp; - } - - virtual void - serialize(vespalib::nbostream &os) const override; + storage::spi::Timestamp getPrevTimestamp() const { return _prevTimestamp; } + void setPrevTimestamp(storage::spi::Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; } - virtual void - deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; uint32_t getSerializedDocSize() const { return _serializedDocSize; } + + // Provided as a hook for tests. + void serializeDocumentOperationOnly(vespalib::nbostream &os) const; }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index e44b7874c12..8a00c739126 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -52,8 +52,7 @@ public: void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; } SerialNum getSerialNum() const { return _serialNum; } virtual void serialize(vespalib::nbostream &os) const = 0; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) = 0; + virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) = 0; virtual vespalib::string toString() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp index a6d34b696ca..efac7297c67 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp @@ -49,6 +49,15 @@ PutOperation::deserialize(vespalib::nbostream &is, _serializedDocSize = oldSize - is.size(); } +void +PutOperation::deserializeDocument(const DocumentTypeRepo &repo) +{ + vespalib::nbostream stream; + _doc->serialize(stream); + auto fixedDoc = std::make_shared<Document>(repo, stream); + _doc = std::move(fixedDoc); +} + vespalib::string PutOperation::toString() const { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h index 7e9dee2cbc0..33330692fab 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h @@ -21,6 +21,7 @@ public: virtual void serialize(vespalib::nbostream &os) const override; virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + void deserializeDocument(const document::DocumentTypeRepo &repo); virtual vespalib::string toString() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index fa5bbee8e6d..3468cc68ced 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -8,6 +8,7 @@ LOG_SETUP(".proton.feedoperation.updateoperation"); using document::BucketId; +using document::DocumentType; using document::DocumentTypeRepo; using document::DocumentUpdate; using storage::spi::Timestamp; @@ -27,51 +28,50 @@ UpdateOperation::UpdateOperation(Type type) } -UpdateOperation::UpdateOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) - : DocumentOperation(type, - bucketId, - timestamp), +UpdateOperation::UpdateOperation(Type type, const BucketId &bucketId, + const Timestamp ×tamp, const DocumentUpdate::SP &upd) + : DocumentOperation(type, bucketId, timestamp), _upd(upd) { } -UpdateOperation::UpdateOperation(const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) +UpdateOperation::UpdateOperation(const BucketId &bucketId, const Timestamp ×tamp, const DocumentUpdate::SP &upd) : UpdateOperation(FeedOperation::UPDATE, bucketId, timestamp, upd) { } +void +UpdateOperation::serializeUpdate(vespalib::nbostream &os) const +{ + assert(getType() == UPDATE); + _upd->serializeHEAD(os); +} + +void +UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) +{ + document::ByteBuffer buf(is.peek(), is.size()); + DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); + is.adjustReadPos(buf.getPos()); + _upd = std::move(update); +} void UpdateOperation::serialize(vespalib::nbostream &os) const { assertValidBucketId(_upd->getId()); DocumentOperation::serialize(os); - if (getType() == FeedOperation::UPDATE_42) { - _upd->serialize42(os); - } else { - _upd->serializeHEAD(os); - } + serializeUpdate(os); } void -UpdateOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &repo) +UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo) { DocumentOperation::deserialize(is, repo); - document::ByteBuffer buf(is.peek(), is.size()); - using Version = DocumentUpdate::SerializeVersion; - Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD); try { - DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version)); - is.adjustReadPos(buf.getPos()); - _upd = update; + deserializeUpdate(is, repo); } catch (document::DocumentTypeNotFoundException &e) { LOG(warning, "Failed deserialize update operation using unknown document type '%s'", e.getDocumentTypeName().c_str()); @@ -80,6 +80,14 @@ UpdateOperation::deserialize(vespalib::nbostream &is, } } +void +UpdateOperation::deserializeUpdate(const DocumentTypeRepo &repo) +{ + vespalib::nbostream stream; + serializeUpdate(stream); + deserializeUpdate(stream, repo); +} + vespalib::string UpdateOperation::toString() const { return make_string("%s(%s, %s)", ((getType() == FeedOperation::UPDATE_42) ? "Update42" : "Update"), @@ -88,12 +96,4 @@ vespalib::string UpdateOperation::toString() const { docArgsToString().c_str()); } -UpdateOperation -UpdateOperation::makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const document::DocumentUpdate::SP &upd) -{ - return UpdateOperation(FeedOperation::UPDATE_42, bucketId, timestamp, upd); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 6e061f79f30..99dcbfbce6c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -3,7 +3,10 @@ #include "documentoperation.h" -namespace document { class DocumentUpdate; } +namespace document { +class DocumentTypeRepo; +class DocumentUpdate; +} namespace proton { @@ -12,24 +15,23 @@ class UpdateOperation : public DocumentOperation private: using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; DocumentUpdateSP _upd; - UpdateOperation(Type type, - const document::BucketId &bucketId, + UpdateOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); + void serializeUpdate(vespalib::nbostream &os) const; + void deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); UpdateOperation(const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); - virtual ~UpdateOperation() {} + ~UpdateOperation() override {} const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; - virtual vespalib::string toString() const override; - static UpdateOperation makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const DocumentUpdateSP &upd); + void deserializeUpdate(const document::DocumentTypeRepo &repo); + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 78084f29742..0dee7adfa49 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC ranking_constants.cpp requestcontext.cpp result_processor.cpp + same_element_builder.cpp search_session.cpp session_manager_explorer.cpp sessionmanager.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 97fc1905f50..268fe63ba4c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -2,6 +2,8 @@ #include "querynodes.h" #include "blueprintbuilder.h" +#include "termdatafromnode.h" +#include "same_element_builder.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> @@ -10,8 +12,7 @@ using namespace search::queryeval; -namespace proton { -namespace matching { +namespace proton::matching { namespace { @@ -99,6 +100,15 @@ private: n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit()); } + void buildSameElement(ProtonSameElement &n) { + SameElementBuilder builder(_requestContext, _context); + for (size_t i = 0; i < n.getChildren().size(); ++i) { + search::query::Node &node = *n.getChildren()[i]; + builder.add_child(node); + } + _result = builder.build(); + } + template <typename NodeType> void buildTerm(NodeType &n) { FieldSpecList indexFields; @@ -124,29 +134,31 @@ private: } protected: - virtual void visit(ProtonAnd &n) override { buildIntermediate(new AndBlueprint(), n); } - virtual void visit(ProtonAndNot &n) override { buildIntermediate(new AndNotBlueprint(), n); } - virtual void visit(ProtonOr &n) override { buildIntermediate(new OrBlueprint(), n); } - virtual void visit(ProtonWeakAnd &n) override { buildWeakAnd(n); } - virtual void visit(ProtonEquiv &n) override { buildEquiv(n); } - virtual void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } - virtual void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } - virtual void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } - - virtual void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } - virtual void visit(ProtonDotProduct &n) override { buildTerm(n); } - virtual void visit(ProtonWandTerm &n) override { buildTerm(n); } - - virtual void visit(ProtonPhrase &n) override { buildTerm(n); } - virtual void visit(ProtonNumberTerm &n) override { buildTerm(n); } - virtual void visit(ProtonLocationTerm &n) override { buildTerm(n); } - virtual void visit(ProtonPrefixTerm &n) override { buildTerm(n); } - virtual void visit(ProtonRangeTerm &n) override { buildTerm(n); } - virtual void visit(ProtonStringTerm &n) override { buildTerm(n); } - virtual void visit(ProtonSubstringTerm &n) override { buildTerm(n); } - virtual void visit(ProtonSuffixTerm &n) override { buildTerm(n); } - virtual void visit(ProtonPredicateQuery &n) override { buildTerm(n); } - virtual void visit(ProtonRegExpTerm &n) override { buildTerm(n); } + void visit(ProtonAnd &n) override { buildIntermediate(new AndBlueprint(), n); } + void visit(ProtonAndNot &n) override { buildIntermediate(new AndNotBlueprint(), n); } + void visit(ProtonOr &n) override { buildIntermediate(new OrBlueprint(), n); } + void visit(ProtonWeakAnd &n) override { buildWeakAnd(n); } + void visit(ProtonEquiv &n) override { buildEquiv(n); } + void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } + void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } + void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } + void visit(ProtonSameElement &n) override { buildSameElement(n); } + + + void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } + void visit(ProtonDotProduct &n) override { buildTerm(n); } + void visit(ProtonWandTerm &n) override { buildTerm(n); } + + void visit(ProtonPhrase &n) override { buildTerm(n); } + void visit(ProtonNumberTerm &n) override { buildTerm(n); } + void visit(ProtonLocationTerm &n) override { buildTerm(n); } + void visit(ProtonPrefixTerm &n) override { buildTerm(n); } + void visit(ProtonRangeTerm &n) override { buildTerm(n); } + void visit(ProtonStringTerm &n) override { buildTerm(n); } + void visit(ProtonSubstringTerm &n) override { buildTerm(n); } + void visit(ProtonSuffixTerm &n) override { buildTerm(n); } + void visit(ProtonPredicateQuery &n) override { buildTerm(n); } + void visit(ProtonRegExpTerm &n) override { buildTerm(n); } public: BlueprintBuilderVisitor(const IRequestContext & requestContext, ISearchContext &context) : @@ -174,5 +186,4 @@ BlueprintBuilder::build(const IRequestContext & requestContext, return result; } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h index eb45a735780..44cd6ffabfd 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h @@ -20,4 +20,3 @@ struct BlueprintBuilder { }; } - diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index c1c4387cf41..1efb74b96ba 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -141,7 +141,7 @@ MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) { template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> uint32_t -MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) +MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange &docid_range) { SearchIterator *search = &tools.search(); search->initRange(docid_range.begin, docid_range.end); @@ -175,12 +175,14 @@ MatchThread::match_loop(MatchTools &tools, HitCollector &hits) uint32_t docsCovered = 0; Context context(matchParams.rankDropLimit, tools, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); - !docid_range.empty() && ! softDoomed; + !docid_range.empty(); docid_range = scheduler.next_range(thread_id)) { - uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); - softDoomed = (lastCovered < docid_range.end); - docsCovered += std::min(lastCovered, docid_range.end) - docid_range.begin; + if (!softDoomed) { + uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); + softDoomed = (lastCovered < docid_range.end); + docsCovered += std::min(lastCovered, docid_range.end) - docid_range.begin; + } } uint32_t matches = context.matches; if (do_limit && context.isBelowLimit()) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index b7ecf149001..5c78ee59b1d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -75,7 +75,7 @@ private: bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> - uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline)); + uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange &docid_range) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> void match_loop(MatchTools &tools, HitCollector &hits) __attribute__((noinline)); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h index ebe4e9b2ad7..a20f648ca1c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matchdatareservevisitor.h @@ -22,13 +22,14 @@ public: template <class TermNode> void visitTerm(TermNode &n) { n.allocateTerms(_mdl); } - virtual void visit(ProtonNodeTypes::Equiv &n) override { + void visit(ProtonNodeTypes::Equiv &n) override { MatchDataReserveVisitor subAllocator(n.children_mdl); for (size_t i = 0; i < n.getChildren().size(); ++i) { n.getChildren()[i]->accept(subAllocator); } n.allocateTerms(_mdl); } + void visit(ProtonNodeTypes::SameElement &) override { } MatchDataReserveVisitor(search::fef::MatchDataLayout &mdl) : _mdl(mdl) {} }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 9181205bb19..8896e4d35c8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -78,8 +78,8 @@ void AddLocationNode(const string &location_str, Node::UP &query_tree, Location } } // namespace -Query::Query() {} -Query::~Query() {} +Query::Query() = default; +Query::~Query() = default; bool Query::buildTree(const vespalib::stringref &stack, const string &location, diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp index 63302424db6..b0d45ee0684 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp @@ -30,7 +30,7 @@ namespace proton::matching { ProtonTermData::ProtonTermData() = default; ProtonTermData::ProtonTermData(const ProtonTermData &) = default; ProtonTermData & ProtonTermData::operator = (const ProtonTermData &) = default; -ProtonTermData::~ProtonTermData() { } +ProtonTermData::~ProtonTermData() = default; void ProtonTermData::setDocumentFrequency(double freq) @@ -42,9 +42,9 @@ ProtonTermData::setDocumentFrequency(double freq) void ProtonTermData::resolve(const ViewResolver &resolver, - const IIndexEnvironment &idxEnv, - const string &view, - bool forceFilter) + const IIndexEnvironment &idxEnv, + const string &view, + bool forceFilter) { std::vector<string> fields; resolver.resolve(((view == "") ? "default" : view), fields); diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h index ca6c8c75310..986c5d2dde9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.h +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.h @@ -104,13 +104,15 @@ struct ProtonTerm : public ProtonTermBase<Base> { ~ProtonTerm() {} }; -typedef search::query::SimpleAnd ProtonAnd; -typedef search::query::SimpleAndNot ProtonAndNot; -typedef search::query::SimpleNear ProtonNear; -typedef search::query::SimpleONear ProtonONear; -typedef search::query::SimpleOr ProtonOr; -typedef search::query::SimpleRank ProtonRank; -typedef search::query::SimpleWeakAnd ProtonWeakAnd; +typedef search::query::SimpleAnd ProtonAnd; +typedef search::query::SimpleAndNot ProtonAndNot; +typedef search::query::SimpleNear ProtonNear; +typedef search::query::SimpleONear ProtonONear; +typedef search::query::SimpleOr ProtonOr; +typedef search::query::SimpleRank ProtonRank; +typedef search::query::SimpleWeakAnd ProtonWeakAnd; +typedef search::query::SimpleSameElement ProtonSameElement; + struct ProtonEquiv final : public ProtonTermBase<search::query::Equiv> { @@ -121,6 +123,7 @@ struct ProtonEquiv final : public ProtonTermBase<search::query::Equiv> typedef ProtonTerm<search::query::LocationTerm> ProtonLocationTerm; typedef ProtonTerm<search::query::NumberTerm> ProtonNumberTerm; typedef ProtonTerm<search::query::Phrase> ProtonPhrase; + typedef ProtonTerm<search::query::PrefixTerm> ProtonPrefixTerm; typedef ProtonTerm<search::query::RangeTerm> ProtonRangeTerm; typedef ProtonTerm<search::query::StringTerm> ProtonStringTerm; @@ -142,6 +145,7 @@ struct ProtonNodeTypes { typedef ProtonONear ONear; typedef ProtonOr Or; typedef ProtonPhrase Phrase; + typedef ProtonSameElement SameElement; typedef ProtonPrefixTerm PrefixTerm; typedef ProtonRangeTerm RangeTerm; typedef ProtonRank Rank; diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp new file mode 100644 index 00000000000..d3a0ec4726f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp @@ -0,0 +1,96 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_builder.h" +#include "querynodes.h" +#include <vespa/searchlib/query/tree/customtypevisitor.h> +#include <vespa/searchlib/queryeval/leaf_blueprints.h> + +using search::queryeval::Blueprint; +using search::queryeval::EmptyBlueprint; +using search::queryeval::FieldSpecList; +using search::queryeval::IRequestContext; +using search::queryeval::SameElementBlueprint; +using search::queryeval::Searchable; + +namespace proton::matching { + +namespace { + +class SameElementBuilderVisitor : public search::query::CustomTypeVisitor<ProtonNodeTypes> +{ +private: + const IRequestContext &_requestContext; + ISearchContext &_context; + SameElementBlueprint &_result; + +public: + SameElementBuilderVisitor(const IRequestContext &requestContext, ISearchContext &context, SameElementBlueprint &result) + : _requestContext(requestContext), + _context(context), + _result(result) {} + + template <class TermNode> + void visitTerm(const TermNode &n) { + if (n.numFields() == 1) { + const ProtonTermData::FieldEntry &field = n.field(0); + assert(field.getFieldId() != search::fef::IllegalFieldId); + assert(field.getHandle() == search::fef::IllegalHandle); + FieldSpecList field_spec; + field_spec.add(_result.getNextChildField(field.field_name, field.getFieldId())); + Searchable &searchable = field.attribute_field ? _context.getAttributes() : _context.getIndexes(); + _result.addTerm(searchable.createBlueprint(_requestContext, field_spec, n)); + } + } + + void visit(ProtonAnd &) override {} + void visit(ProtonAndNot &) override {} + void visit(ProtonNear &) override {} + void visit(ProtonONear &) override {} + void visit(ProtonOr &) override {} + void visit(ProtonRank &) override {} + void visit(ProtonWeakAnd &) override {} + void visit(ProtonSameElement &) override {} + + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPhrase &) override {} + void visit(ProtonEquiv &) override {} + + void visit(ProtonNumberTerm &n) override { visitTerm(n); } + void visit(ProtonLocationTerm &n) override { visitTerm(n); } + void visit(ProtonPrefixTerm &n) override { visitTerm(n); } + void visit(ProtonRangeTerm &n) override { visitTerm(n); } + void visit(ProtonStringTerm &n) override { visitTerm(n); } + void visit(ProtonSubstringTerm &n) override { visitTerm(n); } + void visit(ProtonSuffixTerm &n) override { visitTerm(n); } + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &n) override { visitTerm(n); } +}; + +} // namespace proton::matching::<unnamed> + +SameElementBuilder::SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context) + : _requestContext(requestContext), + _context(context), + _result(std::make_unique<SameElementBlueprint>()) +{ +} + +void +SameElementBuilder::add_child(search::query::Node &node) +{ + SameElementBuilderVisitor visitor(_requestContext, _context, *_result); + node.accept(visitor); +} + +Blueprint::UP +SameElementBuilder::build() +{ + if (!_result || _result->terms().empty()) { + return std::make_unique<EmptyBlueprint>(); + } + return std::move(_result); +} + +} // namespace proton::matching diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h new file mode 100644 index 00000000000..945bb9a97f6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "isearchcontext.h" +#include <vespa/searchlib/query/tree/node.h> +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/same_element_blueprint.h> + +namespace proton::matching { + +class SameElementBuilder +{ +private: + const search::queryeval::IRequestContext &_requestContext; + ISearchContext &_context; + std::unique_ptr<search::queryeval::SameElementBlueprint> _result; +public: + SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context); + void add_child(search::query::Node &node); + search::queryeval::Blueprint::UP build(); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp index 6a9857e1172..596cc7fce1e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp @@ -41,6 +41,8 @@ public: // XXX: unranked equiv not supported _term_data.push_back(&n); } + + virtual void visit(ProtonNodeTypes::SameElement &) override {} }; } // namespace diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp index 26b765cf3cf..3fd4000bf9f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.cpp @@ -4,8 +4,7 @@ #include "querynodes.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> -namespace proton { -namespace matching { +namespace proton::matching { namespace { struct TermDataFromTermVisitor @@ -19,29 +18,30 @@ struct TermDataFromTermVisitor data = &n; } - virtual void visit(ProtonAnd &) override {} - virtual void visit(ProtonAndNot &) override {} - virtual void visit(ProtonNear &) override {} - virtual void visit(ProtonONear &) override {} - virtual void visit(ProtonOr &) override {} - virtual void visit(ProtonRank &) override {} - virtual void visit(ProtonWeakAnd &) override {} - - virtual void visit(ProtonWeightedSetTerm &n) override { visitTerm(n); } - virtual void visit(ProtonDotProduct &n) override { visitTerm(n); } - virtual void visit(ProtonWandTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPhrase &n) override { visitTerm(n); } - virtual void visit(ProtonEquiv &n) override { visitTerm(n); } - - virtual void visit(ProtonNumberTerm &n) override { visitTerm(n); } - virtual void visit(ProtonLocationTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPrefixTerm &n) override { visitTerm(n); } - virtual void visit(ProtonRangeTerm &n) override { visitTerm(n); } - virtual void visit(ProtonStringTerm &n) override { visitTerm(n); } - virtual void visit(ProtonSubstringTerm &n) override { visitTerm(n); } - virtual void visit(ProtonSuffixTerm &n) override { visitTerm(n); } - virtual void visit(ProtonPredicateQuery &) override { } - virtual void visit(ProtonRegExpTerm &n) override { visitTerm(n); } + void visit(ProtonAnd &) override {} + void visit(ProtonAndNot &) override {} + void visit(ProtonNear &) override {} + void visit(ProtonONear &) override {} + void visit(ProtonOr &) override {} + void visit(ProtonRank &) override {} + void visit(ProtonWeakAnd &) override {} + void visit(ProtonSameElement &) override { } + + void visit(ProtonWeightedSetTerm &n) override { visitTerm(n); } + void visit(ProtonDotProduct &n) override { visitTerm(n); } + void visit(ProtonWandTerm &n) override { visitTerm(n); } + void visit(ProtonPhrase &n) override { visitTerm(n); } + void visit(ProtonEquiv &n) override { visitTerm(n); } + + void visit(ProtonNumberTerm &n) override { visitTerm(n); } + void visit(ProtonLocationTerm &n) override { visitTerm(n); } + void visit(ProtonPrefixTerm &n) override { visitTerm(n); } + void visit(ProtonRangeTerm &n) override { visitTerm(n); } + void visit(ProtonStringTerm &n) override { visitTerm(n); } + void visit(ProtonSubstringTerm &n) override { visitTerm(n); } + void visit(ProtonSuffixTerm &n) override { visitTerm(n); } + void visit(ProtonPredicateQuery &) override { } + void visit(ProtonRegExpTerm &n) override { visitTerm(n); } }; } // namespace @@ -53,5 +53,4 @@ termDataFromNode(const search::query::Node &node) return visitor.data; } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h index 3e78ace1900..e445dbcfc50 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h +++ b/searchcore/src/vespa/searchcore/proton/matching/termdatafromnode.h @@ -2,16 +2,13 @@ #pragma once - namespace search { namespace query { class Node; } } -namespace proton { -namespace matching { +namespace proton::matching { class ProtonTermData; const ProtonTermData *termDataFromNode(const search::query::Node &node); -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 2b2849d025c..78733b14aaa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -13,23 +13,6 @@ using search::index::Schema; namespace proton { -FastAccessFeedView::UpdateScope -FastAccessFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - for (const auto &update : upd.getUpdates()) { - const vespalib::string &fieldName = update.getField().getName(); - if (!fastPartialUpdateAttribute(fieldName)) { - updateScope._nonAttributeFields = true; - break; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - /** * NOTE: For both put, update and remove we only need to pass the 'onWriteDone' * instance when we are going to commit as part of handling the operation. @@ -47,9 +30,18 @@ FastAccessFeedView::putAttributes(SerialNum serialNum, search::DocumentIdT lid, void FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) +{ + _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone, onUpdate); +} + +void +FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) { - _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone); + if (_attributeWriter->getHasCompoundAttribute()) { + _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone); + } } void @@ -107,19 +99,4 @@ FastAccessFeedView::sync() _writeService.attributeFieldWriter().sync(); } -bool -FastAccessFeedView::fastPartialUpdateAttribute(const vespalib::string &fieldName) const { - search::AttributeVector *attribute = _attributeWriter->getWritableAttribute(fieldName); - if (attribute == nullptr) { - // Partial update to non-attribute field must update document - return false; - } - search::attribute::BasicType::Type attrType = attribute->getBasicType(); - // Partial update to tensor, predicate or reference attribute - // must update document - return ((attrType != search::attribute::BasicType::Type::PREDICATE) && - (attrType != search::attribute::BasicType::Type::TENSOR) && - (attrType != search::attribute::BasicType::Type::REFERENCE)); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h index e1b0cf83f64..3af97b4ecb9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h @@ -39,14 +39,13 @@ private: const IAttributeWriter::SP _attributeWriter; DocIdLimit &_docIdLimit; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; - void putAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, bool immediateCommit, OnPutDoneType onWriteDone) override; void updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) override; - void removeAttributes(SerialNum serialNum, search::DocumentIdT lid, bool immediateCommit, OnRemoveDoneType onWriteDone) override; @@ -73,8 +72,6 @@ public: void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; void sync() override; - - bool fastPartialUpdateAttribute(const vespalib::string &fieldName) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index ad5372b4af6..90b9bbc7f34 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -96,6 +96,14 @@ void FeedHandler::performPut(FeedToken token, PutOperation &op) { } return; } + /* + * Check if document type repos are equal. DocumentTypeRepoFactory::make + * returns the same document type repo if document type configs are equal, + * thus we can just perform a cheaper identity check here. + */ + if (_repo != op.getDocument()->getRepo()) { + op.deserializeDocument(*_repo); + } storeOperation(op, token); if (token) { token->setResult(make_unique<Result>(), false); @@ -344,6 +352,8 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _feedLock(), _feedState(make_shared<InitState>(getDocTypeName())), _activeFeedView(nullptr), + _repo(nullptr), + _documentType(nullptr), _bucketDBHandler(nullptr), _syncLock(), _syncedSerialNum(0), @@ -408,6 +418,14 @@ void FeedHandler::changeToNormalFeedState() { changeFeedState(make_shared<NormalState>(*this)); } +void +FeedHandler::setActiveFeedView(IFeedView *feedView) +{ + _activeFeedView = feedView; + _repo = feedView->getDocumentTypeRepo().get(); + _documentType = _repo->getDocumentType(_docTypeName.getName()); +} + bool FeedHandler::isDoingReplay() const { return _tlsMgr.isDoingReplay(); @@ -492,22 +510,17 @@ FeedHandler::considerWriteOperationForRejection(FeedToken & token, const FeedOpe } bool -FeedHandler::considerUpdateOperationForRejection(FeedToken &token, const UpdateOperation &op) +FeedHandler::considerUpdateOperationForRejection(FeedToken &token, UpdateOperation &op) { - const auto *repo = _activeFeedView->getDocumentTypeRepo().get(); const auto &update = *op.getUpdate(); /* * Check if document types are equal. DocumentTypeRepoFactory::make returns * the same document type repo if document type configs are equal, thus we * can just perform a cheaper identity check here. */ - if (repo->getDocumentType(_docTypeName.getName()) != &update.getType()) { + if (_documentType != &update.getType()) { try { - vespalib::nbostream stream; - op.serialize(stream); - UpdateOperation checkOp(op.getType()); - vespalib::nbostream checkStream(stream.peek(), stream.size()); - checkOp.deserialize(stream, *repo); + op.deserializeUpdate(*_repo); } catch (document::FieldNotFoundException &e) { if (token) { auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Field not found'", @@ -516,6 +529,14 @@ FeedHandler::considerUpdateOperationForRejection(FeedToken &token, const UpdateO token->fail(); } return true; + } catch (document::DocumentTypeNotFoundException &e) { + auto message = make_string("Update operation rejected for document '%s' of type '%s': 'Uknown document type', expected '%s'", + update.getId().toString().c_str(), + e.getDocumentTypeName().c_str(), + _docTypeName.toString().c_str()); + token->setResult(make_unique<UpdateResult>(Result::TRANSIENT_ERROR, message), false); + token->fail(); + return true; } } return false; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index faf909080d9..34f979b7115 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -90,6 +90,8 @@ private: FeedStateSP _feedState; // used by master write thread tasks IFeedView *_activeFeedView; + const document::DocumentTypeRepo *_repo; + const document::DocumentType *_documentType; bucketdb::IBucketDBHandler *_bucketDBHandler; std::mutex _syncLock; SerialNum _syncedSerialNum; @@ -102,7 +104,7 @@ private: void doHandleOperation(FeedToken token, FeedOperationUP op); bool considerWriteOperationForRejection(FeedToken & token, const FeedOperation &op); - bool considerUpdateOperationForRejection(FeedToken &token, const UpdateOperation &op); + bool considerUpdateOperationForRejection(FeedToken &token, UpdateOperation &op); /** * Delayed execution of feed operations against feed view, in @@ -203,9 +205,7 @@ public: * Update the active feed view. * Always called by the master write thread so locking is not needed. */ - void setActiveFeedView(IFeedView *feedView) { - _activeFeedView = feedView; - } + void setActiveFeedView(IFeedView *feedView); void setBucketDBHandler(bucketdb::IBucketDBHandler *bucketDBHandler) { _bucketDBHandler = bucketDBHandler; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 0c87d24899d..4cda07eee8b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -100,7 +100,7 @@ void SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) { - Document::UP doc = std::move(futureDoc.get()); + const auto &doc = futureDoc.get(); if (doc) { performIndexPut(serialNum, lid, *doc, immediateCommit, onWriteDone); } @@ -118,29 +118,6 @@ SearchableFeedView::performIndexHeartBeat(SerialNum serialNum) _indexWriter->heartBeat(serialNum); } -SearchableFeedView::UpdateScope -SearchableFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - const Schema &schema = *_schema; - for(size_t i(0), m(upd.getUpdates().size()); - !(updateScope._indexedFields && updateScope._nonAttributeFields) && - (i < m); i++) { - const document::FieldUpdate & fu(upd.getUpdates()[i]); - const vespalib::string &name = fu.getField().getName(); - if (schema.isIndexField(name)) { - updateScope._indexedFields = true; - } - if (!fastPartialUpdateAttribute(name)) { - updateScope._nonAttributeFields = true; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - void SearchableFeedView::updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index 81d81a6cc27..ed3ba6740b1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -34,24 +34,19 @@ private: bool hasIndexedFields() const { return _hasIndexedFields; } - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone); void performIndexHeartBeat(SerialNum serialNum); @@ -60,23 +55,17 @@ private: void performSync(); void heartBeatIndexedFields(SerialNum serialNum) override; - virtual void - putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; + void updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - virtual void - updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone) override; - virtual void - removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone) override; - - virtual void - removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone) override; void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone); void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override; @@ -85,7 +74,7 @@ public: SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx, const PersistentParams ¶ms, const FastAccessFeedView::Context &fastUpdateCtx, Context ctx); - virtual ~SearchableFeedView(); + ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index fdaf07dc466..273b97542ef 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -17,6 +17,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> + LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; @@ -300,25 +302,23 @@ StoreOnlyFeedView::heartBeatIndexedFields(SerialNum ) {} void StoreOnlyFeedView::heartBeatAttributes(SerialNum ) {} - -StoreOnlyFeedView::UpdateScope -StoreOnlyFeedView::getUpdateScope(const DocumentUpdate &upd) +void +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate & upd, bool, + OnOperationDoneType, IFieldUpdateCallback & onUpdate) { - UpdateScope updateScope; - if (!upd.getUpdates().empty() || !upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; + for (const auto & fieldUpdate : upd.getUpdates()) { + onUpdate.onUpdateField(fieldUpdate.getField().getName(), nullptr); } - return updateScope; } - void -StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate &, bool, OnOperationDoneType) {} +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) +{ +} void StoreOnlyFeedView::updateIndexedFields(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) { - abort(); // Should never be called. } void @@ -385,6 +385,34 @@ void StoreOnlyFeedView::heartBeatSummary(SerialNum serialNum) { })); } +StoreOnlyFeedView::UpdateScope::UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd) + : _schema(&schema), + _indexedFields(false), + _nonAttributeFields(!upd.getFieldPathUpdates().empty()) +{} + +namespace { + +bool isAttributeUpdateable(const search::AttributeVector *attribute) { + search::attribute::BasicType::Type attrType = attribute->getBasicType(); + // Partial update to tensor, predicate or reference attribute + // must update document + return ((attrType != search::attribute::BasicType::Type::PREDICATE) && + (attrType != search::attribute::BasicType::Type::TENSOR) && + (attrType != search::attribute::BasicType::Type::REFERENCE)); +} +} + +void +StoreOnlyFeedView::UpdateScope::onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) { + if (!_nonAttributeFields && (attr == nullptr || !isAttributeUpdateable(attr))) { + _nonAttributeFields = true; + } + if (!_indexedFields && _schema->isIndexField(fieldName)) { + _indexedFields = true; + } +} + void StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) { if ( ! updOp.getUpdate()) { @@ -417,15 +445,15 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) bool immediateCommit = _commitTimeTracker.needCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), updOp.getUpdate()); - updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone); + UpdateScope updateScope(*_schema, upd); + updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); - UpdateScope updateScope(getUpdateScope(upd)); if (updateScope.hasIndexOrNonAttributeFields()) { PromisedDoc promisedDoc; - FutureDoc futureDoc = promisedDoc.get_future(); + FutureDoc futureDoc = promisedDoc.get_future().share(); _pendingLidTracker.waitForConsumedLid(lid); if (updateScope._indexedFields) { - updateIndexedFields(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); + updateIndexedFields(serialNum, lid, futureDoc, immediateCommit, onWriteDone); } PromisedStream promisedStream; FutureStream futureStream = promisedStream.get_future(); @@ -444,6 +472,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) std::move(promisedDoc), std::move(promisedStream)); }); #pragma GCC diagnostic pop + updateAttributes(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index b106b87c4fe..a11512590f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -9,6 +9,7 @@ #include "searchcontext.h" #include "pendinglidtracker.h" #include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> @@ -33,6 +34,7 @@ class PutDoneContext; class RemoveDoneContext; class CommitTimeTracker; class IGidToLidChangeHandler; +class IFieldUpdateCallback; namespace documentmetastore { class ILidReuseDelayer; } @@ -59,7 +61,7 @@ public: using OnPutDoneType = const std::shared_ptr<PutDoneContext> &; using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &; using FeedTokenUP = std::unique_ptr<FeedToken>; - using FutureDoc = std::future<std::unique_ptr<Document>>; + using FutureDoc = std::shared_future<std::unique_ptr<Document>>; using PromisedDoc = std::promise<std::unique_ptr<Document>>; using FutureStream = std::future<vespalib::nbostream>; using PromisedStream = std::promise<vespalib::nbostream>; @@ -120,18 +122,19 @@ public: }; protected: - struct UpdateScope + class UpdateScope : public IFieldUpdateCallback { + private: + const search::index::Schema *_schema; + public: bool _indexedFields; bool _nonAttributeFields; - UpdateScope() - : _indexedFields(false), - _nonAttributeFields(false) - {} + UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd); bool hasIndexOrNonAttributeFields() const { return _indexedFields || _nonAttributeFields; } + void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) override; }; private: @@ -200,9 +203,10 @@ private: virtual void putIndexedFields(SerialNum serialNum, Lid lid, const DocumentSP &newDoc, bool immediateCommit, OnOperationDoneType onWriteDone); - virtual UpdateScope getUpdateScope(const DocumentUpdate &upd); - virtual void updateAttributes(SerialNum serialNum, Lid lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate); + + virtual void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone); virtual void updateIndexedFields(SerialNum serialNum, Lid lid, FutureDoc doc, diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp index 8c8ea8e6f28..de4cde956d0 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp @@ -179,28 +179,30 @@ private: _result = mixer.mix(); } - virtual void visit(And &) override { } - virtual void visit(AndNot &) override { } - virtual void visit(Or &) override { } - virtual void visit(WeakAnd &) override { } - virtual void visit(Equiv &) override { } - virtual void visit(Rank &) override { } - virtual void visit(Near &) override { } - virtual void visit(ONear &) override { } - - virtual void visit(WeightedSetTerm &n) override { visitTerm(n); } - virtual void visit(DotProduct &n) override { visitTerm(n); } - virtual void visit(WandTerm &n) override { visitTerm(n); } - virtual void visit(Phrase &n) override { visitTerm(n); } - virtual void visit(NumberTerm &n) override { visitTerm(n); } - virtual void visit(LocationTerm &n) override { visitTerm(n); } - virtual void visit(PrefixTerm &n) override { visitTerm(n); } - virtual void visit(RangeTerm &n) override { visitTerm(n); } - virtual void visit(StringTerm &n) override { visitTerm(n); } - virtual void visit(SubstringTerm &n) override { visitTerm(n); } - virtual void visit(SuffixTerm &n) override { visitTerm(n); } - virtual void visit(PredicateQuery &n) override { visitTerm(n); } - virtual void visit(RegExpTerm &n) override { visitTerm(n); } + void visit(And &) override { } + void visit(AndNot &) override { } + void visit(Or &) override { } + void visit(WeakAnd &) override { } + void visit(Equiv &) override { } + void visit(Rank &) override { } + void visit(Near &) override { } + void visit(ONear &) override { } + void visit(SameElement &) override { } + + + void visit(WeightedSetTerm &n) override { visitTerm(n); } + void visit(DotProduct &n) override { visitTerm(n); } + void visit(WandTerm &n) override { visitTerm(n); } + void visit(Phrase &n) override { visitTerm(n); } + void visit(NumberTerm &n) override { visitTerm(n); } + void visit(LocationTerm &n) override { visitTerm(n); } + void visit(PrefixTerm &n) override { visitTerm(n); } + void visit(RangeTerm &n) override { visitTerm(n); } + void visit(StringTerm &n) override { visitTerm(n); } + void visit(SubstringTerm &n) override { visitTerm(n); } + void visit(SuffixTerm &n) override { visitTerm(n); } + void visit(PredicateQuery &n) override { visitTerm(n); } + void visit(RegExpTerm &n) override { visitTerm(n); } public: CreateBlueprintVisitor(const IIndexCollection &indexes, diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 3b321f4a12f..6e4a3ef1e1f 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -194,6 +194,7 @@ vespa_define_module( src/tests/queryeval/multibitvectoriterator src/tests/queryeval/parallel_weak_and src/tests/queryeval/predicate + src/tests/queryeval/same_element src/tests/queryeval/simple_phrase src/tests/queryeval/sourceblender src/tests/queryeval/sparse_vector_benchmark diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxImporter.java index 047d1b187f5..fa1f929cc80 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxImporter.java @@ -1,3 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -13,8 +15,10 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.functions.Rename; import com.yahoo.tensor.functions.TensorFunction; +import com.yahoo.yolean.Exceptions; import onnx.Onnx; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.Collection; @@ -22,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -31,48 +36,64 @@ import java.util.stream.Collectors; */ public class OnnxImporter { - public OnnxModel importModel(String modelPath, String outputNode) { + private static final Logger log = Logger.getLogger(OnnxImporter.class.getName()); + + public OnnxModel importModel(String modelName, File modelDir) { + return importModel(modelName, modelDir.toString()); + } + + public OnnxModel importModel(String modelName, String modelPath) { try (FileInputStream inputStream = new FileInputStream(modelPath)) { Onnx.ModelProto model = Onnx.ModelProto.parseFrom(inputStream); - return importModel(model, outputNode); + return importModel(modelName, model); } catch (IOException e) { throw new IllegalArgumentException("Could not import ONNX model from '" + modelPath + "'", e); } } - public OnnxModel importModel(Onnx.ModelProto model, String outputNode) { - return importGraph(model.getGraph(), outputNode); + public OnnxModel importModel(String modelName, Onnx.ModelProto model) { + return importGraph(modelName, model.getGraph()); } - private static OnnxModel importGraph(Onnx.GraphProto graph, String outputNode) { - OnnxModel model = new OnnxModel(outputNode); + private static OnnxModel importGraph(String modelName, Onnx.GraphProto graph) { + OnnxModel model = new OnnxModel(modelName); OperationIndex index = new OperationIndex(); - OnnxOperation output = importNode(outputNode, graph, index); - output.type().orElseThrow(() -> new IllegalArgumentException("Output of '" + outputNode + "' has no type.")) - .verifyType(getOutputNode(outputNode, graph).getType()); + importNodes(graph, model, index); + verifyOutputTypes(graph, model, index); + findDimensionNames(model, index); + importExpressions(model, index); - findDimensionNames(output); - importExpressions(output, model); + reportWarnings(model, index); return model; } - private static OnnxOperation importNode(String nodeName, Onnx.GraphProto graph, OperationIndex index) { - if (index.alreadyImported(nodeName)) { - return index.get(nodeName); + private static void importNodes(Onnx.GraphProto graph, OnnxModel model, OperationIndex index) { + for (Onnx.ValueInfoProto valueInfo : graph.getOutputList()) { + importNode(valueInfo.getName(), graph, model, index); + } + } + + private static OnnxOperation importNode(String name, Onnx.GraphProto graph, OnnxModel model, OperationIndex index) { + if (index.alreadyImported(name)) { + return index.get(name); } OnnxOperation operation; - if (isArgumentTensor(nodeName, graph)) { - operation = new Argument(getArgumentTensor(nodeName, graph)); - } else if (isConstantTensor(nodeName, graph)) { - operation = new Constant(getConstantTensor(nodeName, graph)); + if (isArgumentTensor(name, graph)) { + operation = new Argument(getArgumentTensor(name, graph)); + model.input(OnnxOperation.namePartOf(name), operation.vespaName()); + } else if (isConstantTensor(name, graph)) { + operation = new Constant(model.name(), getConstantTensor(name, graph)); } else { - Onnx.NodeProto node = getNodeFromGraph(nodeName, graph); - List<OnnxOperation> inputs = importNodeInputs(node, graph, index); + Onnx.NodeProto node = getNodeFromGraph(name, graph); + List<OnnxOperation> inputs = importNodeInputs(node, graph, model, index); operation = OperationMapper.get(node, inputs); + if (isOutputNode(name, graph)) { + model.output(OnnxOperation.namePartOf(name), operation.vespaName()); + } } - index.put(nodeName, operation); + index.put(operation.vespaName(), operation); return operation; } @@ -113,8 +134,11 @@ public class OnnxImporter { private static Onnx.ValueInfoProto getOutputNode(String name, Onnx.GraphProto graph) { for (Onnx.ValueInfoProto valueInfo : graph.getOutputList()) { - Onnx.NodeProto node = getNodeFromGraph(valueInfo.getName(), graph); - if (node.getName().equals(name)) { + if (valueInfo.getName().equals(name)) { + return valueInfo; + } + String nodeName = OnnxOperation.namePartOf(valueInfo.getName()); + if (nodeName.equals(name)) { return valueInfo; } } @@ -123,18 +147,34 @@ public class OnnxImporter { private static List<OnnxOperation> importNodeInputs(Onnx.NodeProto node, Onnx.GraphProto graph, + OnnxModel model, OperationIndex index) { return node.getInputList().stream() - .map(nodeName -> importNode(nodeName, graph, index)) + .map(nodeName -> importNode(nodeName, graph, model, index)) .collect(Collectors.toList()); } + private static void verifyOutputTypes(Onnx.GraphProto graph, OnnxModel model, OperationIndex index) { + for (String outputName : model.outputs().values()) { + OnnxOperation operation = index.get(outputName); + Onnx.ValueInfoProto onnxNode = getOutputNode(outputName, graph); + operation.type().orElseThrow( + () -> new IllegalArgumentException("Output of '" + outputName + "' has no type.")) + .verifyType(onnxNode.getType()); + } + } + + /** Find dimension names to avoid excessive renaming while evaluating the model. */ - private static void findDimensionNames(OnnxOperation output) { + private static void findDimensionNames(OnnxModel model, OperationIndex index) { DimensionRenamer renamer = new DimensionRenamer(); - addDimensionNameConstraints(output, renamer); + for (String output : model.outputs().values()) { + addDimensionNameConstraints(index.get(output), renamer); + } renamer.solve(); - renameDimensions(output, renamer); + for (String output : model.outputs().values()) { + renameDimensions(index.get(output), renamer); + } } private static void addDimensionNameConstraints(OnnxOperation operation, DimensionRenamer renamer) { @@ -151,10 +191,17 @@ public class OnnxImporter { } } - private static void importExpressions(OnnxOperation output, OnnxModel model) { - Optional<TensorFunction> function = importExpression(output, model); - if (!function.isPresent()) { - throw new IllegalArgumentException("No valid output function could be found."); + private static void importExpressions(OnnxModel model, OperationIndex index) { + for (String outputName : model.outputs().values()) { + try { + Optional<TensorFunction> function = importExpression(index.get(outputName), model); + if (!function.isPresent()) { + model.skippedOutput(outputName, "No valid output function could be found."); + } + } + catch (IllegalArgumentException e) { + model.skippedOutput(outputName, Exceptions.toMessageString(e)); + } } } @@ -167,7 +214,7 @@ public class OnnxImporter { } importInputExpressions(operation, model); importRankingExpression(operation, model); - importInputExpression(operation, model); + importArgumentExpression(operation, model); return operation.function(); } @@ -204,7 +251,7 @@ public class OnnxImporter { if (!model.expressions().containsKey(name)) { TensorFunction function = operation.function().get(); - if (name.equals(model.output())) { + if (model.outputs().containsKey(name)) { OrderedTensorType operationType = operation.type().get(); OrderedTensorType standardNamingType = OrderedTensorType.standardType(operationType); if ( ! operationType.equals(standardNamingType)) { @@ -228,7 +275,7 @@ public class OnnxImporter { } } - private static void importInputExpression(OnnxOperation operation, OnnxModel model) { + private static void importArgumentExpression(OnnxOperation operation, OnnxModel model) { if (operation.isInput()) { // All inputs must have dimensions with standard naming convention: d0, d1, ... OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get()); @@ -237,6 +284,20 @@ public class OnnxImporter { } } + private static void reportWarnings(OnnxModel model, OperationIndex index) { + for (String output : model.outputs().values()) { + reportWarnings(model, index.get(output)); + } + } + + private static void reportWarnings(OnnxModel model, OnnxOperation operation) { + for (String warning : operation.warnings()) { + model.importWarning(warning); + } + for (OnnxOperation input : operation.inputs()) { + reportWarnings(model, input); + } + } private static Onnx.NodeProto getNodeFromGraph(String nodeName, Onnx.GraphProto graph) { boolean hasPortNumber = nodeName.contains(":"); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxModel.java index df108fcbbe7..bd53afefc3f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxModel.java @@ -1,3 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -14,29 +16,73 @@ import java.util.regex.Pattern; /** * The result of importing an ONNX model into Vespa. * + * @author bratseth * @author lesters */ public class OnnxModel { - public OnnxModel(String outputNode) { - this.output = outputNode; + private static final Pattern nameRegexp = Pattern.compile("[A-Za-z0-9_]*"); + + private final String name; + + public OnnxModel(String name) { + if ( ! nameRegexp.matcher(name).matches()) + throw new IllegalArgumentException("A TensorFlow model name can only contain [A-Za-z0-9_], but is '" + + name + "'"); + this.name = name; } - private final String output; + /** Returns the name of this model, which can only contain the characters in [A-Za-z0-9_] */ + public String name() { return name; } + + private final Map<String, String> inputs = new HashMap<>(); + private final Map<String, String> outputs = new HashMap<>(); + private final Map<String, String> skippedOutputs = new HashMap<>(); + private final List<String> importWarnings = new ArrayList<>(); + private final Map<String, TensorType> arguments = new HashMap<>(); private final Map<String, Tensor> smallConstants = new HashMap<>(); private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); + private final Map<String, RankingExpression> macros = new HashMap<>(); private final Map<String, TensorType> requiredMacros = new HashMap<>(); + void input(String inputName, String argumentName) { inputs.put(inputName, argumentName); } + void output(String name, String expressionName) { outputs.put(name, expressionName); } + void skippedOutput(String name, String reason) { skippedOutputs.put(name, reason); } + void importWarning(String warning) { importWarnings.add(warning); } void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); } void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } + void macro(String name, RankingExpression expression) { macros.put(name, expression); } void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); } - /** Return the name of the output node for this model */ - public String output() { return output; } + /** + * Returns an immutable map of the inputs (evaluation context) of this. This is a map from input name + * to argument (Placeholder) name in the owner of this + */ + public Map<String, String> inputs() { return Collections.unmodifiableMap(inputs); } + + /** Returns arguments().get(inputs.get(name)), e.g the type of the argument this input references */ + public TensorType inputArgument(String inputName) { return arguments().get(inputs.get(inputName)); } + + /** Returns an immutable list of the expression names of this */ + public Map<String, String> outputs() { return Collections.unmodifiableMap(outputs); } + + /** + * Returns an immutable list of the outputs of this which could not be imported, + * with a string detailing the reason for each + */ + public Map<String, String> skippedOutputs() { return Collections.unmodifiableMap(skippedOutputs); } + + /** + * Returns an immutable list of possibly non-fatal warnings encountered during import. + */ + public List<String> importWarnings() { return Collections.unmodifiableList(importWarnings); } + + /** Returns expressions().get(outputs.get(outputName)), e.g the expression this output references */ + public RankingExpression outputExpression(String outputName) { return expressions().get(outputs.get(outputName)); } /** Returns an immutable map of the arguments (inputs) of this */ public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); } @@ -57,6 +103,9 @@ public class OnnxModel { */ public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } + /** Returns an immutable map of macros that are part of this model */ + public Map<String, RankingExpression> macros() { return Collections.unmodifiableMap(macros); } + /** Returns an immutable map of the macros that must be provided by the environment running this model */ public Map<String, TensorType> requiredMacros() { return Collections.unmodifiableMap(requiredMacros); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OperationMapper.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OperationMapper.java index 3ee3f6aa32e..12090145d3a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OperationMapper.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OperationMapper.java @@ -1,3 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx.importer; import com.yahoo.searchlib.rankingexpression.integration.onnx.importer.operations.Join; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OrderedTensorType.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OrderedTensorType.java index f6e117bfd74..812e9b8d678 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OrderedTensorType.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/OrderedTensorType.java @@ -1,4 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx.importer; import com.yahoo.tensor.TensorType; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/TensorConverter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/TensorConverter.java index 1c5fef456cb..2912db03b5f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/TensorConverter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/TensorConverter.java @@ -1,3 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx.importer; import com.google.protobuf.ByteString; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/Constant.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/Constant.java index ab650bf8d77..13043a61a8e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/Constant.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/Constant.java @@ -15,18 +15,19 @@ import java.util.Optional; public class Constant extends OnnxOperation { + final String modelName; final Onnx.TensorProto tensorProto; - public Constant(Onnx.TensorProto tensorProto) { + public Constant(String modelName, Onnx.TensorProto tensorProto) { super(null, Collections.emptyList()); + this.modelName = modelName; this.tensorProto = tensorProto; } - /** todo: Constant names are prefixed by "modelName_" to avoid name conflicts between models */ + /** Constant names are prefixed by "modelName_" to avoid name conflicts between models */ @Override public String vespaName() { -// return modelName() + "_" + super.vespaName(); - return vespaName(tensorProto.getName()); + return modelName + "_" + vespaName(tensorProto.getName()); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/OnnxOperation.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/OnnxOperation.java index 2c8003f5951..30f7b4f4711 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/OnnxOperation.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/importer/operations/OnnxOperation.java @@ -1,3 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.searchlib.rankingexpression.integration.onnx.importer.operations; import com.yahoo.searchlib.rankingexpression.Reference; @@ -92,7 +94,7 @@ public abstract class OnnxOperation { /** Retrieve the valid Vespa name of this node */ public String vespaName() { return vespaName(node.getName()); } - public String vespaName(String name) { return name != null ? name.replace('/', '_').replace(':','_') : null; } + public String vespaName(String name) { return name != null ? namePartOf(name).replace('/', '_') : null; } /** Retrieve the list of warnings produced during its lifetime */ public List<String> warnings() { return Collections.unmodifiableList(importWarnings); } @@ -116,4 +118,22 @@ public abstract class OnnxOperation { return verifyInputs(expected, OnnxOperation::function); } + /** + * A method signature input and output has the form name:index. + * This returns the name part without the index. + */ + public static String namePartOf(String name) { + name = name.startsWith("^") ? name.substring(1) : name; + return name.split(":")[0]; + } + + /** + * This return the output index part. Indexes are used for nodes with + * multiple outputs. + */ + public static int indexPartOf(String name) { + int i = name.indexOf(":"); + return i < 0 ? 0 : Integer.parseInt(name.substring(i + 1)); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/package-info.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/package-info.java new file mode 100644 index 00000000000..5cff8b03d40 --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/onnx/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * ONNX integration + */ +@ExportPackage +package com.yahoo.searchlib.rankingexpression.integration.onnx; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java index 4ec23f98fc5..e3c72830095 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java @@ -255,7 +255,7 @@ public class TensorFlowImporter { } catch (ParseException e) { throw new RuntimeException("Tensorflow function " + function + - " cannot be parsed as a ranking expression", e); + " cannot be parsed as a ranking expression", e); } } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/ConcatV2.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/ConcatV2.java index a394662800e..4f5d61d75f9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/ConcatV2.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/ConcatV2.java @@ -28,12 +28,12 @@ public class ConcatV2 extends TensorFlowOperation { TensorFlowOperation concatDimOp = inputs.get(inputs.size() - 1); // ConcatV2: concat dimension is the last input if (!concatDimOp.getConstantValue().isPresent()) { throw new IllegalArgumentException("ConcatV2 in " + node.getName() + ": " + - "concat dimension must be a constant."); + "concat dimension must be a constant."); } Tensor concatDimTensor = concatDimOp.getConstantValue().get().asTensor(); if (concatDimTensor.type().rank() != 0) { throw new IllegalArgumentException("ConcatV2 in " + node.getName() + ": " + - "concat dimension must be a scalar."); + "concat dimension must be a scalar."); } OrderedTensorType aType = inputs.get(0).type().get(); @@ -45,7 +45,7 @@ public class ConcatV2 extends TensorFlowOperation { OrderedTensorType bType = inputs.get(i).type().get(); if (bType.rank() != aType.rank()) { throw new IllegalArgumentException("ConcatV2 in " + node.getName() + ": " + - "inputs must have save rank."); + "inputs must have save rank."); } for (int j = 0; j < aType.rank(); ++j) { long dimSizeA = aType.dimensions().get(j).size().orElse(-1L); @@ -54,7 +54,7 @@ public class ConcatV2 extends TensorFlowOperation { concatDimSize += dimSizeB; } else if (dimSizeA != dimSizeB) { throw new IllegalArgumentException("ConcatV2 in " + node.getName() + ": " + - "input dimension " + j + " differs in input tensors."); + "input dimension " + j + " differs in input tensors."); } } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/Placeholder.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/Placeholder.java index eb4b615b434..1619c11427a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/Placeholder.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/Placeholder.java @@ -53,5 +53,4 @@ public class Placeholder extends TensorFlowOperation { return false; } - } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/PlaceholderWithDefault.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/PlaceholderWithDefault.java index f74d1d6cb75..65ce7f00e34 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/PlaceholderWithDefault.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/importer/operations/PlaceholderWithDefault.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.integration.tensorflow.importer.operations; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.importer.OrderedTensorType; import com.yahoo.tensor.functions.TensorFunction; import org.tensorflow.framework.NodeDef; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxMnistSoftmaxImportTestCase.java index e118c2b885a..4b68cd40a08 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/onnx/OnnxMnistSoftmaxImportTestCase.java @@ -24,18 +24,18 @@ public class OnnxMnistSoftmaxImportTestCase { @Test public void testMnistSoftmaxImport() throws IOException { - OnnxModel model = new OnnxImporter().importModel("src/test/files/integration/onnx/mnist_softmax/mnist_softmax.onnx", "add"); + OnnxModel model = new OnnxImporter().importModel("test", "src/test/files/integration/onnx/mnist_softmax/mnist_softmax.onnx"); // Check constants assertEquals(2, model.largeConstants().size()); - Tensor constant0 = model.largeConstants().get("Variable_0"); + Tensor constant0 = model.largeConstants().get("test_Variable"); assertNotNull(constant0); assertEquals(new TensorType.Builder().indexed("d2", 784).indexed("d1", 10).build(), constant0.type()); assertEquals(7840, constant0.size()); - Tensor constant1 = model.largeConstants().get("Variable_1_0"); + Tensor constant1 = model.largeConstants().get("test_Variable_1"); assertNotNull(constant1); assertEquals(new TensorType.Builder().indexed("d1", 10).build(), constant1.type()); @@ -43,15 +43,15 @@ public class OnnxMnistSoftmaxImportTestCase { // Check required macros (inputs) assertEquals(1, model.requiredMacros().size()); - assertTrue(model.requiredMacros().containsKey("Placeholder_0")); + assertTrue(model.requiredMacros().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.requiredMacros().get("Placeholder_0")); + model.requiredMacros().get("Placeholder")); // Check outputs - RankingExpression output = model.expressions().get("add"); + RankingExpression output = model.outputExpression("add"); assertNotNull(output); assertEquals("add", output.getName()); - assertEquals("join(reduce(join(rename(Placeholder_0, (d0, d1), (d0, d2)), constant(Variable_0), f(a,b)(a * b)), sum, d2), constant(Variable_1_0), f(a,b)(a + b))", + assertEquals("join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(test_Variable), f(a,b)(a * b)), sum, d2), constant(test_Variable_1), f(a,b)(a + b))", output.getRoot().toString()); } @@ -62,7 +62,7 @@ public class OnnxMnistSoftmaxImportTestCase { Tensor argument = placeholderArgument(); Tensor tensorFlowResult = evaluateTensorFlowModel(tfModelPath, argument, "Placeholder", "add"); - Tensor onnxResult = evaluateOnnxModel(onnxModelPath, argument, "Placeholder_0", "add"); + Tensor onnxResult = evaluateOnnxModel(onnxModelPath, argument, "Placeholder", "add"); assertEquals("Operation 'add' produces equal results", tensorFlowResult, onnxResult); } @@ -74,7 +74,7 @@ public class OnnxMnistSoftmaxImportTestCase { } private Tensor evaluateOnnxModel(String path, Tensor argument, String input, String output) { - OnnxModel model = new OnnxImporter().importModel(path, output); + OnnxModel model = new OnnxImporter().importModel("test", path); return evaluateExpression(model.expressions().get(output), contextFrom(model), argument, input); } diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index e8e16ffcc98..0a02824e77a 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -301,24 +301,24 @@ TEST_F("Strict iterator handles seek outside of LID space", ArrayValueFixture) { EXPECT_TRUE(iter->isAtEnd()); } -TEST_F("cmp() performs GID mapping and forwards to target attribute", SingleValueFixture) { +TEST_F("matches() performs GID mapping and forwards to target attribute", SingleValueFixture) { auto ctx = f.create_context(word_term("5678")); - EXPECT_FALSE(ctx->cmp(DocId(2))); - EXPECT_TRUE(ctx->cmp(DocId(3))); - EXPECT_FALSE(ctx->cmp(DocId(4))); - EXPECT_TRUE(ctx->cmp(DocId(5))); + EXPECT_FALSE(ctx->matches(DocId(2))); + EXPECT_TRUE(ctx->matches(DocId(3))); + EXPECT_FALSE(ctx->matches(DocId(4))); + EXPECT_TRUE(ctx->matches(DocId(5))); } -TEST_F("cmp(weight) performs GID mapping and forwards to target attribute", WsetValueFixture) { +TEST_F("matches(weight) performs GID mapping and forwards to target attribute", WsetValueFixture) { auto ctx = f.create_context(word_term("foo")); int32_t weight = 0; - EXPECT_FALSE(ctx->cmp(DocId(1), weight)); + EXPECT_FALSE(ctx->matches(DocId(1), weight)); EXPECT_EQUAL(0, weight); // Unchanged - EXPECT_TRUE(ctx->cmp(DocId(2), weight)); + EXPECT_TRUE(ctx->matches(DocId(2), weight)); EXPECT_EQUAL(-5, weight); - EXPECT_TRUE(ctx->cmp(DocId(6), weight)); + EXPECT_TRUE(ctx->matches(DocId(6), weight)); EXPECT_EQUAL(42, weight); } diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp index cbc86b02ada..77504c4ab3c 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/attribute/multistringattribute.h> +#include <vespa/searchlib/attribute/elementiterator.h> #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> @@ -193,12 +194,8 @@ private: // test search iterator functionality - void testStrictSearchIterator(SearchContext & threeHits, - SearchContext & noHits, - const IteratorTester & typeTester); - void testNonStrictSearchIterator(SearchContext & threeHits, - SearchContext & noHits, - const IteratorTester & typeTester); + void testStrictSearchIterator(SearchContext & threeHits, SearchContext & noHits, const IteratorTester & typeTester); + void testNonStrictSearchIterator(SearchContext & threeHits, SearchContext & noHits, const IteratorTester & typeTester); void fillForSearchIteratorTest(IntegerAttribute * ia); void fillForSemiNibbleSearchIteratorTest(IntegerAttribute * ia); void testSearchIterator(); @@ -206,17 +203,20 @@ private: // test search iterator unpacking void fillForSearchIteratorUnpackingTest(IntegerAttribute * ia, bool extra); - void testSearchIteratorUnpacking(const AttributePtr & ptr, - SearchContext & sc, - bool extra, - bool strict); + void testSearchIteratorUnpacking(const AttributePtr & ptr, SearchContext & sc, bool extra, bool strict) { + sc.fetchPostings(strict); + for (bool withElementId : {false, true}) { + testSearchIteratorUnpacking(ptr, sc, extra, strict, withElementId); + } + } + void testSearchIteratorUnpacking(const AttributePtr & ptr, SearchContext & sc, + bool extra, bool strict, bool withElementId); void testSearchIteratorUnpacking(); // test range search template <typename VectorType> - void performRangeSearch(const VectorType & vec, const vespalib::string & term, - const DocSet & expected); + void performRangeSearch(const VectorType & vec, const vespalib::string & term, const DocSet & expected); template <typename VectorType, typename ValueType> void testRangeSearch(const AttributePtr & ptr, uint32_t numDocs, std::vector<ValueType> values); void testRangeSearch(); @@ -224,8 +224,7 @@ private: // test case insensitive search - void performCaseInsensitiveSearch(const StringAttribute & vec, const vespalib::string & term, - const DocSet & expected); + void performCaseInsensitiveSearch(const StringAttribute & vec, const vespalib::string & term, const DocSet & expected); void testCaseInsensitiveSearch(const AttributePtr & ptr); void testCaseInsensitiveSearch(); void testRegexSearch(const AttributePtr & ptr); @@ -252,25 +251,19 @@ private: void requireThatSearchIsWorkingAfterLoadAndClearDoc(); template <typename VectorType, typename ValueType> - void requireThatSearchIsWorkingAfterUpdates(const vespalib::string & name, - const Config & cfg, - ValueType value1, - ValueType value2); + void requireThatSearchIsWorkingAfterUpdates(const vespalib::string & name, const Config & cfg, + ValueType value1, ValueType value2); void requireThatSearchIsWorkingAfterUpdates(); void requireThatFlagAttributeIsWorkingWhenNewDocsAreAdded(); template <typename VectorType, typename ValueType> - void requireThatInvalidSearchTermGivesZeroHits(const vespalib::string & name, - const Config & cfg, - ValueType value); + void requireThatInvalidSearchTermGivesZeroHits(const vespalib::string & name, const Config & cfg, ValueType value); void requireThatInvalidSearchTermGivesZeroHits(); void requireThatFlagAttributeHandlesTheByteRange(); - void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, - const Config &cfg, - int64_t maxValue); + void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, const Config &cfg, int64_t maxValue); void requireThatOutOfBoundsSearchTermGivesZeroHits(); // init maps with config objects @@ -620,21 +613,30 @@ void SearchContextTest::testSearch(const ConfigMap & cfgs) { template<typename T, typename A> class Verifier : public search::test::SearchIteratorVerifier { public: - Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, const Config & cfg); + Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, + const Config & cfg, bool withElementId); ~Verifier(); - SearchIterator::UP create(bool strict) const override { - return _sc->createIterator(&_dummy, strict); + SearchIterator::UP + create(bool strict) const override { + auto search = _sc->createIterator(&_dummy, strict); + if (_withElementId) { + search = std::make_unique<attribute::ElementIterator>(std::move(search), *_sc, _dummy); + } + return search; } private: mutable TermFieldMatchData _dummy; - AttributePtr _attribute; + const bool _withElementId; + AttributePtr _attribute; SearchContextPtr _sc; }; template<typename T, typename A> -Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, const Config & cfg) - :_attribute(AttributeFactory::createAttribute(name + "-initrange", cfg)), - _sc() +Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vespalib::string & name, + const Config & cfg, bool withElementId) + : _withElementId(withElementId), + _attribute(AttributeFactory::createAttribute(name + "-initrange", cfg)), + _sc() { SearchContextTest::addDocs(*_attribute, getDocIdLimit()); for (uint32_t doc : getExpectedDocIds()) { @@ -648,15 +650,18 @@ Verifier<T, A>::Verifier(T key, const vespalib::string & keyAsString, const vesp } template<typename T, typename A> -Verifier<T, A>::~Verifier() {} +Verifier<T, A>::~Verifier() = default; template<typename T, typename A> void SearchContextTest::testSearchIterator(T key, const vespalib::string &keyAsString, const ConfigMap &cfgs) { - for (const auto & cfg : cfgs) { - Verifier<T, A> verifier(key, keyAsString, cfg.first, cfg.second); - verifier.verify(); + for (bool withElementId : {false, true} ) { + for (const auto & cfg : cfgs) { + Verifier<T, A> verifier(key, keyAsString, cfg.first, cfg.second, withElementId); + verifier.verify(); + } } + } void SearchContextTest::testSearchIteratorConformance() { @@ -935,13 +940,10 @@ SearchContextTest::fillForSearchIteratorUnpackingTest(IntegerAttribute * ia, } void -SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, - SearchContext & sc, - bool extra, - bool strict) +SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, SearchContext & sc, + bool extra, bool strict, bool withElementId) { - LOG(info, - "testSearchIteratorUnpacking: vector '%s'", attr->getName().c_str()); + LOG(info, "testSearchIteratorUnpacking: vector '%s'", attr->getName().c_str()); TermFieldMatchData md; md.reset(100); @@ -950,8 +952,10 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, pos.setElementWeight(100); md.appendPosition(pos); - sc.fetchPostings(strict); SearchBasePtr sb = sc.createIterator(&md, strict); + if (withElementId) { + sb = std::make_unique<attribute::ElementIterator>(std::move(sb), sc, md); + } sb->initFullRange(); std::vector<int32_t> weights(3); @@ -980,12 +984,30 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, sb->unpack(2); EXPECT_EQUAL(sb->getDocId(), 2u); EXPECT_EQUAL(md.getDocId(), 2u); - EXPECT_EQUAL(md.getWeight(), weights[1]); + if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { + EXPECT_EQUAL(2, md.end()- md.begin()); + EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); + EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); + EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); + } else { + EXPECT_EQUAL(md.getWeight(), weights[1]); + } sb->unpack(3); EXPECT_EQUAL(sb->getDocId(), 3u); EXPECT_EQUAL(md.getDocId(), 3u); - EXPECT_EQUAL(md.getWeight(), weights[2]); + if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { + EXPECT_EQUAL(3, md.end()- md.begin()); + EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); + EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); + EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); + EXPECT_EQUAL(md.begin()[2].getElementId(), 2u); + EXPECT_EQUAL(md.begin()[2].getElementWeight(), 1); + } else { + EXPECT_EQUAL(md.getWeight(), weights[2]); + } if (extra) { sb->unpack(4); EXPECT_EQUAL(sb->getDocId(), 4u); @@ -1894,7 +1916,7 @@ SearchContextTest::SearchContextTest() : initStringConfig(); } -SearchContextTest::~SearchContextTest() {} +SearchContextTest::~SearchContextTest() = default; int SearchContextTest::Main() diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 5aabb328354..a941694d375 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for customtypevisitor. -#include <vespa/log/log.h> -LOG_SETUP("customtypevisitor_test"); - #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("customtypevisitor_test"); + using std::string; using namespace search::query; @@ -39,6 +39,7 @@ struct MyNear : Near { MyNear() : Near(1) {} }; struct MyONear : ONear { MyONear() : ONear(1) {} }; struct MyOr : Or {}; struct MyPhrase : Phrase { MyPhrase() : Phrase("view", 0, Weight(42)) {} }; +struct MySameElement : SameElement { MySameElement() : SameElement("view") {} }; struct MyRank : Rank {}; struct MyNumberTerm : InitTerm<NumberTerm> {}; struct MyLocationTerm : InitTerm<LocationTerm> {}; @@ -64,6 +65,7 @@ struct MyQueryNodeTypes { typedef MyONear ONear; typedef MyOr Or; typedef MyPhrase Phrase; + typedef MySameElement SameElement; typedef MyPrefixTerm PrefixTerm; typedef MyRangeTerm RangeTerm; typedef MyRank Rank; @@ -89,27 +91,28 @@ public: template <typename T> void setVisited() { isVisited<T>() = true; } - virtual void visit(MyAnd &) override { setVisited<MyAnd>(); } - virtual void visit(MyAndNot &) override { setVisited<MyAndNot>(); } - virtual void visit(MyEquiv &) override { setVisited<MyEquiv>(); } - virtual void visit(MyNumberTerm &) override { setVisited<MyNumberTerm>(); } - virtual void visit(MyLocationTerm &) override { setVisited<MyLocationTerm>(); } - virtual void visit(MyNear &) override { setVisited<MyNear>(); } - virtual void visit(MyONear &) override { setVisited<MyONear>(); } - virtual void visit(MyOr &) override { setVisited<MyOr>(); } - virtual void visit(MyPhrase &) override { setVisited<MyPhrase>(); } - virtual void visit(MyPrefixTerm &) override { setVisited<MyPrefixTerm>(); } - virtual void visit(MyRangeTerm &) override { setVisited<MyRangeTerm>(); } - virtual void visit(MyRank &) override { setVisited<MyRank>(); } - virtual void visit(MyStringTerm &) override { setVisited<MyStringTerm>(); } - virtual void visit(MySubstrTerm &) override { setVisited<MySubstrTerm>(); } - virtual void visit(MySuffixTerm &) override { setVisited<MySuffixTerm>(); } - virtual void visit(MyWeakAnd &) override { setVisited<MyWeakAnd>(); } - virtual void visit(MyWeightedSetTerm &) override { setVisited<MyWeightedSetTerm>(); } - virtual void visit(MyDotProduct &) override { setVisited<MyDotProduct>(); } - virtual void visit(MyWandTerm &) override { setVisited<MyWandTerm>(); } - virtual void visit(MyPredicateQuery &) override { setVisited<MyPredicateQuery>(); } - virtual void visit(MyRegExpTerm &) override { setVisited<MyRegExpTerm>(); } + void visit(MyAnd &) override { setVisited<MyAnd>(); } + void visit(MyAndNot &) override { setVisited<MyAndNot>(); } + void visit(MyEquiv &) override { setVisited<MyEquiv>(); } + void visit(MyNumberTerm &) override { setVisited<MyNumberTerm>(); } + void visit(MyLocationTerm &) override { setVisited<MyLocationTerm>(); } + void visit(MyNear &) override { setVisited<MyNear>(); } + void visit(MyONear &) override { setVisited<MyONear>(); } + void visit(MyOr &) override { setVisited<MyOr>(); } + void visit(MyPhrase &) override { setVisited<MyPhrase>(); } + void visit(MySameElement &) override { setVisited<MySameElement>(); } + void visit(MyPrefixTerm &) override { setVisited<MyPrefixTerm>(); } + void visit(MyRangeTerm &) override { setVisited<MyRangeTerm>(); } + void visit(MyRank &) override { setVisited<MyRank>(); } + void visit(MyStringTerm &) override { setVisited<MyStringTerm>(); } + void visit(MySubstrTerm &) override { setVisited<MySubstrTerm>(); } + void visit(MySuffixTerm &) override { setVisited<MySuffixTerm>(); } + void visit(MyWeakAnd &) override { setVisited<MyWeakAnd>(); } + void visit(MyWeightedSetTerm &) override { setVisited<MyWeightedSetTerm>(); } + void visit(MyDotProduct &) override { setVisited<MyDotProduct>(); } + void visit(MyWandTerm &) override { setVisited<MyWandTerm>(); } + void visit(MyPredicateQuery &) override { setVisited<MyPredicateQuery>(); } + void visit(MyRegExpTerm &) override { setVisited<MyRegExpTerm>(); } }; template <class T> diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index 9e82c6ea5ec..f8922c54a4e 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for query_visitor. -#include <vespa/log/log.h> -LOG_SETUP("query_visitor_test"); - #include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/queryvisitor.h> @@ -11,6 +8,9 @@ LOG_SETUP("query_visitor_test"); #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> +LOG_SETUP("query_visitor_test"); + using namespace search::query; namespace { @@ -43,29 +43,28 @@ public: return b; } - virtual void visit(And &) override { isVisited<And>() = true; } - virtual void visit(AndNot &) override { isVisited<AndNot>() = true; } - virtual void visit(Equiv &) override { isVisited<Equiv>() = true; } - virtual void visit(NumberTerm &) override { isVisited<NumberTerm>() = true; } - virtual void visit(LocationTerm &) override { isVisited<LocationTerm>() = true; } - virtual void visit(Near &) override { isVisited<Near>() = true; } - virtual void visit(ONear &) override { isVisited<ONear>() = true; } - virtual void visit(Or &) override { isVisited<Or>() = true; } - virtual void visit(Phrase &) override { isVisited<Phrase>() = true; } - virtual void visit(PrefixTerm &) override { isVisited<PrefixTerm>() = true; } - virtual void visit(RangeTerm &) override { isVisited<RangeTerm>() = true; } - virtual void visit(Rank &) override { isVisited<Rank>() = true; } - virtual void visit(StringTerm &) override { isVisited<StringTerm>() = true; } - virtual void visit(SubstringTerm &) override { isVisited<SubstringTerm>() = true; } - virtual void visit(SuffixTerm &) override { isVisited<SuffixTerm>() = true; } - virtual void visit(WeakAnd &) override { isVisited<WeakAnd>() = true; } - virtual void visit(WeightedSetTerm &) override - { isVisited<WeightedSetTerm>() = true; } - virtual void visit(DotProduct &) override { isVisited<DotProduct>() = true; } - virtual void visit(WandTerm &) override { isVisited<WandTerm>() = true; } - virtual void visit(PredicateQuery &) override - { isVisited<PredicateQuery>() = true; } - virtual void visit(RegExpTerm &) override { isVisited<RegExpTerm>() = true; } + void visit(And &) override { isVisited<And>() = true; } + void visit(AndNot &) override { isVisited<AndNot>() = true; } + void visit(Equiv &) override { isVisited<Equiv>() = true; } + void visit(NumberTerm &) override { isVisited<NumberTerm>() = true; } + void visit(LocationTerm &) override { isVisited<LocationTerm>() = true; } + void visit(Near &) override { isVisited<Near>() = true; } + void visit(ONear &) override { isVisited<ONear>() = true; } + void visit(Or &) override { isVisited<Or>() = true; } + void visit(Phrase &) override { isVisited<Phrase>() = true; } + void visit(SameElement &) override { isVisited<SameElement>() = true; } + void visit(PrefixTerm &) override { isVisited<PrefixTerm>() = true; } + void visit(RangeTerm &) override { isVisited<RangeTerm>() = true; } + void visit(Rank &) override { isVisited<Rank>() = true; } + void visit(StringTerm &) override { isVisited<StringTerm>() = true; } + void visit(SubstringTerm &) override { isVisited<SubstringTerm>() = true; } + void visit(SuffixTerm &) override { isVisited<SuffixTerm>() = true; } + void visit(WeakAnd &) override { isVisited<WeakAnd>() = true; } + void visit(WeightedSetTerm &) override { isVisited<WeightedSetTerm>() = true; } + void visit(DotProduct &) override { isVisited<DotProduct>() = true; } + void visit(WandTerm &) override { isVisited<WandTerm>() = true; } + void visit(PredicateQuery &) override { isVisited<PredicateQuery>() = true; } + void visit(RegExpTerm &) override { isVisited<RegExpTerm>() = true; } }; template <class T> @@ -84,27 +83,20 @@ void Test::requireThatAllNodesCanBeVisited() { checkVisit<ONear>(new SimpleONear(0)); checkVisit<Or>(new SimpleOr); checkVisit<Phrase>(new SimplePhrase("field", 0, Weight(42))); - checkVisit<WeightedSetTerm>( - new SimpleWeightedSetTerm("field", 0, Weight(42))); + checkVisit<SameElement>(new SimpleSameElement("field")); + checkVisit<WeightedSetTerm>(new SimpleWeightedSetTerm("field", 0, Weight(42))); checkVisit<DotProduct>(new SimpleDotProduct("field", 0, Weight(42))); - checkVisit<WandTerm>( - new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); + checkVisit<WandTerm>(new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); checkVisit<Rank>(new SimpleRank); - checkVisit<NumberTerm>( - new SimpleNumberTerm("0.42", "field", 0, Weight(0))); + checkVisit<NumberTerm>(new SimpleNumberTerm("0.42", "field", 0, Weight(0))); const Location location(Point(10, 10), 20, 0); - checkVisit<LocationTerm>( - new SimpleLocationTerm(location, "field", 0, Weight(0))); + checkVisit<LocationTerm>(new SimpleLocationTerm(location, "field", 0, Weight(0))); checkVisit<PrefixTerm>(new SimplePrefixTerm("t", "field", 0, Weight(0))); - checkVisit<RangeTerm>( - new SimpleRangeTerm(Range(0, 1), "field", 0, Weight(0))); + checkVisit<RangeTerm>(new SimpleRangeTerm(Range(0, 1), "field", 0, Weight(0))); checkVisit<StringTerm>(new SimpleStringTerm("t", "field", 0, Weight(0))); - checkVisit<SubstringTerm>( - new SimpleSubstringTerm("t", "field", 0, Weight(0))); + checkVisit<SubstringTerm>(new SimpleSubstringTerm("t", "field", 0, Weight(0))); checkVisit<SuffixTerm>(new SimpleSuffixTerm("t", "field", 0, Weight(0))); - checkVisit<PredicateQuery>( - new SimplePredicateQuery(PredicateQueryTerm::UP(), - "field", 0, Weight(0))); + checkVisit<PredicateQuery>(new SimplePredicateQuery(PredicateQueryTerm::UP(), "field", 0, Weight(0))); checkVisit<RegExpTerm>(new SimpleRegExpTerm("t", "field", 0, Weight(0))); } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index b968f794833..0f05419c4e5 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -1,22 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for querybuilder. -#include <vespa/log/log.h> -LOG_SETUP("querybuilder_test"); - #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/searchlib/query/tree/customtypevisitor.h> -#include <vespa/searchlib/query/tree/intermediatenodes.h> #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/querybuilder.h> -#include <vespa/searchlib/query/tree/querytreecreator.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> -#include <vespa/searchlib/query/tree/termnodes.h> -#include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <string> + +#include <vespa/log/log.h> +LOG_SETUP("querybuilder_test"); +#include <vespa/searchlib/query/tree/querytreecreator.h> using std::string; using search::SimpleQueryStackDumpIterator; @@ -52,7 +48,7 @@ PredicateQueryTerm::UP getPredicateQueryTerm() { template <class NodeTypes> Node::UP createQueryTree() { QueryBuilder<NodeTypes> builder; - builder.addAnd(9); + builder.addAnd(10); { builder.addRank(2); { @@ -109,6 +105,12 @@ Node::UP createQueryTree() { builder.addStringTerm(str[2], view[2], id[2], weight[2]); } builder.addRegExpTerm(str[5], view[5], id[5], weight[5]); + builder.addSameElement(3, view[4]); + { + builder.addStringTerm(str[4], view[4], id[4], weight[5]); + builder.addStringTerm(str[5], view[5], id[5], weight[6]); + builder.addStringTerm(str[6], view[6], id[6], weight[7]); + } } Node::UP node = builder.build(); ASSERT_TRUE(node.get()); @@ -146,6 +148,7 @@ void checkQueryTreeTypes(Node *node) { //typedef typename NodeTypes::NumberTerm FloatTrm; typedef typename NodeTypes::Near Near; typedef typename NodeTypes::ONear ONear; + typedef typename NodeTypes::SameElement SameElement; typedef typename NodeTypes::Or Or; typedef typename NodeTypes::Phrase Phrase; typedef typename NodeTypes::PrefixTerm PrefixTerm; @@ -165,7 +168,7 @@ void checkQueryTreeTypes(Node *node) { ASSERT_TRUE(node); And *and_node = dynamic_cast<And *>(node); ASSERT_TRUE(and_node); - EXPECT_EQUAL(9u, and_node->getChildren().size()); + EXPECT_EQUAL(10u, and_node->getChildren().size()); Rank *rank = dynamic_cast<Rank *>(and_node->getChildren()[0]); @@ -176,22 +179,18 @@ void checkQueryTreeTypes(Node *node) { ASSERT_TRUE(near); EXPECT_EQUAL(2u, near->getChildren().size()); EXPECT_EQUAL(distance, near->getDistance()); - StringTerm *string_term = - dynamic_cast<StringTerm *>(near->getChildren()[0]); + StringTerm *string_term = dynamic_cast<StringTerm *>(near->getChildren()[0]); EXPECT_TRUE(checkTerm(string_term, str[0], view[0], id[0], weight[0])); - SubstringTerm *substring_term = - dynamic_cast<SubstringTerm *>(near->getChildren()[1]); + SubstringTerm *substring_term = dynamic_cast<SubstringTerm *>(near->getChildren()[1]); EXPECT_TRUE(checkTerm(substring_term, str[1], view[1], id[1], weight[1])); ONear *onear = dynamic_cast<ONear *>(rank->getChildren()[1]); ASSERT_TRUE(onear); EXPECT_EQUAL(2u, onear->getChildren().size()); EXPECT_EQUAL(distance, onear->getDistance()); - SuffixTerm *suffix_term = - dynamic_cast<SuffixTerm *>(onear->getChildren()[0]); + SuffixTerm *suffix_term = dynamic_cast<SuffixTerm *>(onear->getChildren()[0]); EXPECT_TRUE(checkTerm(suffix_term, str[2], view[2], id[2], weight[2])); - PrefixTerm *prefix_term = - dynamic_cast<PrefixTerm *>(onear->getChildren()[1]); + PrefixTerm *prefix_term = dynamic_cast<PrefixTerm *>(onear->getChildren()[1]); EXPECT_TRUE(checkTerm(prefix_term, str[3], view[3], id[3], weight[3])); @@ -224,26 +223,20 @@ void checkQueryTreeTypes(Node *node) { AndNot *and_not = dynamic_cast<AndNot *>(or_node->getChildren()[2]); ASSERT_TRUE(and_not); EXPECT_EQUAL(2u, and_not->getChildren().size()); - NumberTerm *integer_term = - dynamic_cast<NumberTerm *>(and_not->getChildren()[0]); + NumberTerm *integer_term = dynamic_cast<NumberTerm *>(and_not->getChildren()[0]); EXPECT_TRUE(checkTerm(integer_term, int1, view[7], id[7], weight[7])); - NumberTerm *float_term = - dynamic_cast<NumberTerm *>(and_not->getChildren()[1]); - EXPECT_TRUE(checkTerm(float_term, float1, view[8], id[8], weight[8], - false)); + NumberTerm *float_term = dynamic_cast<NumberTerm *>(and_not->getChildren()[1]); + EXPECT_TRUE(checkTerm(float_term, float1, view[8], id[8], weight[8], false)); - RangeTerm *range_term = - dynamic_cast<RangeTerm *>(and_node->getChildren()[2]); + RangeTerm *range_term = dynamic_cast<RangeTerm *>(and_node->getChildren()[2]); ASSERT_TRUE(range_term); EXPECT_TRUE(checkTerm(range_term, range, view[9], id[9], weight[9])); - LocationTerm *loc_term = - dynamic_cast<LocationTerm *>(and_node->getChildren()[3]); + LocationTerm *loc_term = dynamic_cast<LocationTerm *>(and_node->getChildren()[3]); ASSERT_TRUE(loc_term); EXPECT_TRUE(checkTerm(loc_term, location, view[10], id[10], weight[10])); - WeakAnd *wand = dynamic_cast<WeakAnd *>(and_node->getChildren()[4]); ASSERT_TRUE(wand != 0); EXPECT_EQUAL(123u, wand->getMinHits()); @@ -253,15 +246,12 @@ void checkQueryTreeTypes(Node *node) { string_term = dynamic_cast<StringTerm *>(wand->getChildren()[1]); EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[5])); - PredicateQuery *predicateQuery = - dynamic_cast<PredicateQuery *>(and_node->getChildren()[5]); + PredicateQuery *predicateQuery = dynamic_cast<PredicateQuery *>(and_node->getChildren()[5]); ASSERT_TRUE(predicateQuery); PredicateQueryTerm::UP pqt(new PredicateQueryTerm); - EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), - view[3], id[3], weight[3])); + EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), view[3], id[3], weight[3])); - DotProduct *dotProduct = - dynamic_cast<DotProduct *>(and_node->getChildren()[6]); + DotProduct *dotProduct = dynamic_cast<DotProduct *>(and_node->getChildren()[6]); ASSERT_TRUE(dotProduct); EXPECT_EQUAL(3u, dotProduct->getChildren().size()); string_term = dynamic_cast<StringTerm *>(dotProduct->getChildren()[0]); @@ -282,9 +272,20 @@ void checkQueryTreeTypes(Node *node) { string_term = dynamic_cast<StringTerm *>(wandTerm->getChildren()[1]); EXPECT_TRUE(checkTerm(string_term, str[2], view[2], id[2], weight[2])); - RegExpTerm *regexp_term = - dynamic_cast<RegExpTerm *>(and_node->getChildren()[8]); + RegExpTerm *regexp_term = dynamic_cast<RegExpTerm *>(and_node->getChildren()[8]); EXPECT_TRUE(checkTerm(regexp_term, str[5], view[5], id[5], weight[5])); + + SameElement *same = dynamic_cast<SameElement *>(and_node->getChildren()[9]); + ASSERT_TRUE(same != nullptr); + EXPECT_EQUAL(view[4], same->getView()); + EXPECT_EQUAL(3u, same->getChildren().size()); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[0]); + EXPECT_TRUE(checkTerm(string_term, str[4], view[4], id[4], weight[5])); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[1]); + EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[6])); + string_term = dynamic_cast<StringTerm *>(same->getChildren()[2]); + EXPECT_TRUE(checkTerm(string_term, str[6], view[6], id[6], weight[7])); + } struct AbstractTypes { @@ -294,6 +295,7 @@ struct AbstractTypes { typedef search::query::LocationTerm LocationTerm; typedef search::query::Near Near; typedef search::query::ONear ONear; + typedef search::query::SameElement SameElement; typedef search::query::Or Or; typedef search::query::Phrase Phrase; typedef search::query::PrefixTerm PrefixTerm; @@ -333,9 +335,9 @@ struct MyNear : Near { MyNear(size_t dist) : Near(dist) {} }; struct MyONear : ONear { MyONear(size_t dist) : ONear(dist) {} }; struct MyWeakAnd : WeakAnd { MyWeakAnd(uint32_t minHits, const vespalib::string & v) : WeakAnd(minHits, v) {} }; struct MyOr : Or {}; -struct MyPhrase : Phrase { - MyPhrase(const string &f, int32_t i, Weight w) : Phrase(f, i, w) {} -}; +struct MyPhrase : Phrase { MyPhrase(const string &f, int32_t i, Weight w) : Phrase(f, i, w) {}}; +struct MySameElement : SameElement { MySameElement(const string &f) : SameElement(f) {}}; + struct MyWeightedSetTerm : WeightedSetTerm { MyWeightedSetTerm(const string &f, int32_t i, Weight w) : WeightedSetTerm(f, i, w) {} }; @@ -404,6 +406,7 @@ struct MyQueryNodeTypes { typedef MyONear ONear; typedef MyOr Or; typedef MyPhrase Phrase; + typedef MySameElement SameElement; typedef MyPrefixTerm PrefixTerm; typedef MyRangeTerm RangeTerm; typedef MyRank Rank; @@ -556,8 +559,7 @@ TEST("require that Query Tree Creator Can Create Queries From Stack") { string stackDump = StackDumpCreator::create(*node); SimpleQueryStackDumpIterator iterator(stackDump); - Node::UP new_node = - QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); + Node::UP new_node = QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); checkQueryTreeTypes<SimpleQueryNodeTypes>(new_node.get()); } diff --git a/searchlib/src/tests/queryeval/same_element/CMakeLists.txt b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt new file mode 100644 index 00000000000..97034c53376 --- /dev/null +++ b/searchlib/src/tests/queryeval/same_element/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_same_element_test_app TEST + SOURCES + same_element_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_same_element_test_app COMMAND searchlib_same_element_test_app) diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp new file mode 100644 index 00000000000..d89883bc417 --- /dev/null +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/queryeval/leaf_blueprints.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/queryeval/same_element_blueprint.h> + +using namespace search::fef; +using namespace search::queryeval; + +std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResult> &children) { + auto result = std::make_unique<SameElementBlueprint>(); + for (size_t i = 0; i < children.size(); ++i) { + uint32_t field_id = i; + vespalib::string field_name = vespalib::make_string("f%u", field_id); + FieldSpec field = result->getNextChildField(field_name, field_id); + result->addTerm(std::make_unique<FakeBlueprint>(field, children[i])); + } + return result; +} + +Blueprint::UP finalize(Blueprint::UP bp, bool strict) { + Blueprint::UP result = Blueprint::optimize(std::move(bp)); + result->fetchPostings(strict); + result->freeze(); + return result; +} + +SimpleResult find_matches(const std::vector<FakeResult> &children) { + auto md = MatchData::makeTestInstance(0, 0); + auto bp = finalize(make_blueprint(children), false); + auto search = bp->createSearch(*md, false); + return SimpleResult().search(*search, 1000); +} + +FakeResult make_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) { + FakeResult result; + uint32_t pos_should_be_ignored = 0; + for (const auto &doc: match_data) { + result.doc(doc.first); + for (const auto &elem: doc.second) { + result.elem(elem).pos(++pos_should_be_ignored); + } + } + return result; +} + +TEST("require that simple match can be found") { + auto a = make_result({{5, {1,3,7}}}); + auto b = make_result({{5, {3,5,10}}}); + SimpleResult result = find_matches({a, b}); + SimpleResult expect({5}); + EXPECT_EQUAL(result, expect); +} + +TEST("require that children must match within same element") { + auto a = make_result({{5, {1,3,7}}}); + auto b = make_result({{5, {2,5,10}}}); + SimpleResult result = find_matches({a, b}); + SimpleResult expect; + EXPECT_EQUAL(result, expect); +} + +TEST("require that strict iterator seeks to next hit") { + auto md = MatchData::makeTestInstance(0, 0); + auto a = make_result({{5, {1,2}}, {7, {1,2}}, {8, {1,2}}, {9, {1,2}}}); + auto b = make_result({{5, {3}}, {6, {1,2}}, {7, {2,4}}, {9, {1}}}); + auto bp = finalize(make_blueprint({a,b}), true); + auto search = bp->createSearch(*md, true); + search->initRange(1, 1000); + EXPECT_LESS(search->getDocId(), 1u); + EXPECT_TRUE(!search->seek(1)); + EXPECT_EQUAL(search->getDocId(), 7u); + EXPECT_TRUE(search->seek(9)); + EXPECT_EQUAL(search->getDocId(), 9u); + EXPECT_TRUE(!search->seek(10)); + EXPECT_TRUE(search->isAtEnd()); +} + +TEST("require that results are estimated appropriately") { + auto a = make_result({{5, {0}}, {5, {0}}, {5, {0}}}); + auto b = make_result({{5, {0}}, {5, {0}}}); + auto c = make_result({{5, {0}}, {5, {0}}, {5, {0}}, {5, {0}}}); + auto bp = finalize(make_blueprint({a,b,c}), true); + EXPECT_EQUAL(bp->getState().estimate().estHits, 2u); +} + +TEST("require that children are sorted") { + auto a = make_result({{5, {0}}, {5, {0}}, {5, {0}}}); + auto b = make_result({{5, {0}}, {5, {0}}}); + auto c = make_result({{5, {0}}, {5, {0}}, {5, {0}}, {5, {0}}}); + auto bp = finalize(make_blueprint({a,b,c}), true); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[0]->getState().estimate().estHits, 2u); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[1]->getState().estimate().estHits, 3u); + EXPECT_EQUAL(dynamic_cast<SameElementBlueprint&>(*bp).terms()[2]->getState().estimate().estHits, 4u); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index d3a99dc439a..8ad4578b6c1 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -244,6 +244,7 @@ StackDumpIteratorTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_NUMTERM, "foo", "[0;22]")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_PREFIXTERM, "bar", "baz")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "bar")); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_SAME_ELEMENT, 3, "bar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_OR, 2)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 3)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_RANK, 5)); diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 794e0b2bdf6..58218ecfd65 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(searchlib_attribute OBJECT defines.cpp diversity.cpp dociditerator.cpp + elementiterator.cpp enumattribute.cpp enumattributesaver.cpp enumcomparator.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 60b8b1603c8..712288f9a1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -65,6 +65,8 @@ using search::queryeval::ParallelWeakAndBlueprint; using search::queryeval::PredicateBlueprint; using search::queryeval::SearchIterator; using search::queryeval::Searchable; +using search::queryeval::SimpleLeafBlueprint; +using search::queryeval::ComplexLeafBlueprint; using search::queryeval::WeightedSetTermBlueprint; using vespalib::geo::ZCurve; using vespalib::string; @@ -77,18 +79,15 @@ namespace { /** * Blueprint for creating regular, stack-based attribute iterators. **/ -class AttributeFieldBlueprint : - public search::queryeval::SimpleLeafBlueprint +class AttributeFieldBlueprint : public SimpleLeafBlueprint { private: ISearchContext::UP _search_context; - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const string &query_stack, - const attribute::SearchContextParams ¶ms) + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, + const string &query_stack, const attribute::SearchContextParams ¶ms) : SimpleLeafBlueprint(field), - _search_context(attribute.createSearchContext(QueryTermDecoder::decodeTerm(query_stack), params).release()) + _search_context(attribute.createSearchContext(QueryTermDecoder::decodeTerm(query_stack), params)) { uint32_t estHits = _search_context->approximateHits(); HitEstimate estimate(estHits, estHits == 0); @@ -96,40 +95,30 @@ private: } public: - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const string &query_stack) - : AttributeFieldBlueprint(field, - attribute, - query_stack, - attribute::SearchContextParams() - .useBitVector(field.isFilter())) - { - } - - AttributeFieldBlueprint(const FieldSpec &field, - const IAttributeVector &attribute, - const IAttributeVector &diversity, - const string &query_stack, - size_t diversityCutoffGroups, - bool diversityCutoffStrict) - : AttributeFieldBlueprint(field, - attribute, - query_stack, + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const string &query_stack) + : AttributeFieldBlueprint(field, attribute, query_stack, + attribute::SearchContextParams().useBitVector(field.isFilter())) + {} + + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, + const IAttributeVector &diversity, const string &query_stack, + size_t diversityCutoffGroups, bool diversityCutoffStrict) + : AttributeFieldBlueprint(field, attribute, query_stack, attribute::SearchContextParams() .diversityAttribute(&diversity) .useBitVector(field.isFilter()) .diversityCutoffGroups(diversityCutoffGroups) .diversityCutoffStrict(diversityCutoffStrict)) - { - } + {} - SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { + SearchIterator::UP + createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); return _search_context->createIterator(tfmda[0], strict); } - void fetchPostings(bool strict) override { + void + fetchPostings(bool strict) override { _search_context->fetchPostings(strict); } @@ -139,7 +128,7 @@ public: void AttributeFieldBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const { - search::queryeval::LeafBlueprint::visitMembers(visitor); + LeafBlueprint::visitMembers(visitor); visit(visitor, "attribute", _search_context->attributeName()); } @@ -149,11 +138,10 @@ template <bool is_strict> struct LocationPreFilterIterator : public OrLikeSearch<is_strict, NoUnpack> { LocationPreFilterIterator(const std::vector<SearchIterator *> &children) : OrLikeSearch<is_strict, NoUnpack>(children, NoUnpack()) {} - virtual void doUnpack(uint32_t) override {} + void doUnpack(uint32_t) override {} }; -class LocationPreFilterBlueprint : - public search::queryeval::ComplexLeafBlueprint +class LocationPreFilterBlueprint : public ComplexLeafBlueprint { private: const IAttributeVector &_attribute; @@ -171,8 +159,8 @@ public: const IAttributeVector &attr(_attribute); for (auto it(rangeVector.begin()), mt(rangeVector.end()); it != mt; it++) { const ZCurve::Range &r(*it); - search::query::Range qr(r.min(), r.max()); - search::query::SimpleRangeTerm rt(qr, "", 0, search::query::Weight(0)); + query::Range qr(r.min(), r.max()); + query::SimpleRangeTerm rt(qr, "", 0, query::Weight(0)); string stack(StackDumpCreator::create(rt)); _rangeSearches.push_back(attr.createSearchContext(QueryTermDecoder::decodeTerm(stack), attribute::SearchContextParams())); @@ -191,22 +179,21 @@ public: bool should_use() const { return _should_use; } - virtual SearchIterator::UP + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { std::vector<SearchIterator *> children; for (auto it(_rangeSearches.begin()), mt(_rangeSearches.end()); it != mt; it++) { - children.push_back((*it)->createIterator(tfmda[0], - strict).release()); + children.push_back((*it)->createIterator(tfmda[0], strict).release()); } if (strict) { - return SearchIterator::UP(new LocationPreFilterIterator<true>(children)); + return std::make_unique<LocationPreFilterIterator<true>>(children); } else { - return SearchIterator::UP(new LocationPreFilterIterator<false>(children)); + return std::make_unique<LocationPreFilterIterator<false>>(children); } } - virtual void fetchPostings(bool strict) override { + void fetchPostings(bool strict) override { for (size_t i(0); i < _rangeSearches.size(); i++) { _rangeSearches[i]->fetchPostings(strict); } @@ -215,12 +202,11 @@ public: //----------------------------------------------------------------------------- -class LocationPostFilterBlueprint : - public search::queryeval::ComplexLeafBlueprint +class LocationPostFilterBlueprint : public ComplexLeafBlueprint { private: const IAttributeVector &_attribute; - search::common::Location _location; + common::Location _location; public: LocationPostFilterBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) @@ -235,46 +221,43 @@ public: setEstimate(estimate); } - const search::common::Location &location() const { return _location; } + const common::Location &location() const { return _location; } - virtual SearchIterator::UP + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &, bool strict) const override { - unsigned int num_docs = _attribute.getNumDocs(); - return SearchIterator::UP(FastS_AllocLocationIterator(num_docs, strict, _location)); + return FastS_AllocLocationIterator(_attribute.getNumDocs(), strict, _location); } }; //----------------------------------------------------------------------------- -Blueprint::UP make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { - LocationPostFilterBlueprint *post_filter = new LocationPostFilterBlueprint(field, attribute, loc); - Blueprint::UP post_filter_bp(post_filter); - const search::common::Location &location = post_filter->location(); +Blueprint::UP +make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { + auto post_filter = std::make_unique<LocationPostFilterBlueprint>(field, attribute, loc); + const common::Location &location = post_filter->location(); if (location.getMinX() > location.getMaxX() || location.getMinY() > location.getMaxY()) { - return Blueprint::UP(new queryeval::EmptyBlueprint(field)); + return std::make_unique<queryeval::EmptyBlueprint>(field); } ZCurve::RangeVector rangeVector = ZCurve::find_ranges( location.getMinX(), location.getMinY(), location.getMaxX(), location.getMaxY()); - LocationPreFilterBlueprint *pre_filter = new LocationPreFilterBlueprint(field, attribute, rangeVector); - Blueprint::UP pre_filter_bp(pre_filter); + auto pre_filter = std::make_unique<LocationPreFilterBlueprint>(field, attribute, rangeVector); if (!pre_filter->should_use()) { - return post_filter_bp; + return post_filter; } - AndBlueprint *root = new AndBlueprint(); - Blueprint::UP root_bp(root); - root->addChild(std::move(pre_filter_bp)); - root->addChild(std::move(post_filter_bp)); - return root_bp; + auto root = std::make_unique<AndBlueprint>(); + root->addChild(std::move(pre_filter)); + root->addChild(std::move(post_filter)); + return root; } //----------------------------------------------------------------------------- template <typename SearchType> -class DirectWeightedSetBlueprint : public search::queryeval::ComplexLeafBlueprint +class DirectWeightedSetBlueprint : public ComplexLeafBlueprint { private: HitEstimate _estimate; @@ -283,8 +266,7 @@ private: const IDocumentWeightAttribute &_attr; public: - DirectWeightedSetBlueprint(const FieldSpec &field, - const IDocumentWeightAttribute &attr, size_t size_hint) + DirectWeightedSetBlueprint(const FieldSpec &field, const IDocumentWeightAttribute &attr, size_t size_hint) : ComplexLeafBlueprint(field), _estimate(), _weights(), @@ -315,7 +297,7 @@ public: { assert(tfmda.size() == 1); if (_terms.size() == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } std::vector<DocumentWeightIterator> iterators; const size_t numChildren = _terms.size(); @@ -329,7 +311,7 @@ public: //----------------------------------------------------------------------------- -class DirectWandBlueprint : public search::queryeval::ComplexLeafBlueprint +class DirectWandBlueprint : public queryeval::ComplexLeafBlueprint { private: HitEstimate _estimate; @@ -342,12 +324,8 @@ private: const IDocumentWeightAttribute &_attr; public: - DirectWandBlueprint(const FieldSpec &field, - const IDocumentWeightAttribute &attr, - uint32_t scoresToTrack, - queryeval::wand::score_t scoreThreshold, - double thresholdBoostFactor, - size_t size_hint) + DirectWandBlueprint(const FieldSpec &field, const IDocumentWeightAttribute &attr, uint32_t scoresToTrack, + queryeval::wand::score_t scoreThreshold, double thresholdBoostFactor, size_t size_hint) : ComplexLeafBlueprint(field), _estimate(), _scores(scoresToTrack), @@ -380,20 +358,19 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); if (_terms.size() == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } - return search::queryeval::ParallelWeakAndSearch::create(*tfmda[0], - queryeval::ParallelWeakAndSearch::MatchParams(_scores, - _scoreThreshold, - _thresholdBoostFactor, - _scoresAdjustFrequency).setDocIdLimit(get_docid_limit()), + return queryeval::ParallelWeakAndSearch::create(*tfmda[0], + queryeval::ParallelWeakAndSearch::MatchParams(_scores, _scoreThreshold, + _thresholdBoostFactor, _scoresAdjustFrequency) + .setDocIdLimit(get_docid_limit()), _weights, _terms, _attr, strict); } }; //----------------------------------------------------------------------------- -class DirectAttributeBlueprint : public search::queryeval::SimpleLeafBlueprint +class DirectAttributeBlueprint : public queryeval::SimpleLeafBlueprint { private: vespalib::string _attrName; @@ -401,8 +378,7 @@ private: IDocumentWeightAttribute::LookupResult _dict_entry; public: - DirectAttributeBlueprint(const FieldSpec &field, - const vespalib::string & name, + DirectAttributeBlueprint(const FieldSpec &field, const vespalib::string & name, const IDocumentWeightAttribute &attr, const vespalib::string &term) : SimpleLeafBlueprint(field), _attrName(name), @@ -415,13 +391,13 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { assert(tfmda.size() == 1); if (_dict_entry.posting_size == 0) { - return SearchIterator::UP(new search::queryeval::EmptySearch()); + return std::make_unique<queryeval::EmptySearch>(); } - return SearchIterator::UP(new queryeval::DocumentWeightSearchIterator(*tfmda[0], _attr, _dict_entry)); + return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); } void visitMembers(vespalib::ObjectVisitor &visitor) const override { - search::queryeval::LeafBlueprint::visitMembers(visitor); + LeafBlueprint::visitMembers(visitor); visit(visitor, "attribute", _attrName); } }; @@ -429,10 +405,7 @@ public: //----------------------------------------------------------------------------- bool check_valid_diversity_attr(const IAttributeVector *attr) { - if (attr == nullptr) { - return false; - } - if (attr->hasMultiValue()) { + if ((attr == nullptr) || attr->hasMultiValue()) { return false; } return (attr->hasEnum() || attr->isIntegerType() || attr->isFloatingPointType()); @@ -452,10 +425,8 @@ private: const IDocumentWeightAttribute *_dwa; public: - CreateBlueprintVisitor(Searchable &searchable, - const IRequestContext &requestContext, - const FieldSpec &field, - const IAttributeVector &attr) + CreateBlueprintVisitor(Searchable &searchable, const IRequestContext &requestContext, + const FieldSpec &field, const IAttributeVector &attr) : CreateBlueprintVisitorHelper(searchable, field, requestContext), _field(field), _attr(attr), @@ -464,11 +435,11 @@ public: template <class TermNode> void visitTerm(TermNode &n, bool simple = false) { if (simple && (_dwa != nullptr) && !_field.isFilter() && n.isRanked()) { - vespalib::string term = search::queryeval::termAsString(n); - setResult(make_UP(new DirectAttributeBlueprint(_field, _attr.getName(), *_dwa, term))); + vespalib::string term = queryeval::termAsString(n); + setResult(std::make_unique<DirectAttributeBlueprint>(_field, _attr.getName(), *_dwa, term)); } else { const string stack = StackDumpCreator::create(n); - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, stack))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, stack)); } } @@ -478,14 +449,12 @@ public: } void visitPredicate(PredicateQuery &query) { - const PredicateAttribute *attr = - dynamic_cast<const PredicateAttribute *>(&_attr); + const PredicateAttribute *attr = dynamic_cast<const PredicateAttribute *>(&_attr); if (!attr) { - LOG(warning, "Trying to apply a PredicateQuery node to a " - "non-predicate attribute."); - setResult(Blueprint::UP(new queryeval::EmptyBlueprint(_field))); + LOG(warning, "Trying to apply a PredicateQuery node to a non-predicate attribute."); + setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } else { - setResult(Blueprint::UP(new PredicateBlueprint( _field, *attr, query))); + setResult(std::make_unique<PredicateBlueprint>( _field, *attr, query)); } } @@ -495,31 +464,31 @@ public: void visit(RangeTerm &n) override { const string stack = StackDumpCreator::create(n); - const string term = search::queryeval::termAsString(n); - search::QueryTermSimple parsed_term(term, search::QueryTermSimple::WORD); + const string term = queryeval::termAsString(n); + QueryTermSimple parsed_term(term, QueryTermSimple::WORD); if (parsed_term.getMaxPerGroup() > 0) { const IAttributeVector *diversity(getRequestContext().getAttribute(parsed_term.getDiversityAttribute())); if (check_valid_diversity_attr(diversity)) { - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, *diversity, stack, - parsed_term.getDiversityCutoffGroups(), - parsed_term.getDiversityCutoffStrict()))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, *diversity, stack, + parsed_term.getDiversityCutoffGroups(), + parsed_term.getDiversityCutoffStrict())); } else { - setResult(Blueprint::UP(new queryeval::EmptyBlueprint(_field))); + setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } } else { - setResult(make_UP(new AttributeFieldBlueprint(_field, _attr, stack))); + setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, stack)); } } void visit(StringTerm & n) override { visitTerm(n, true); } void visit(SubstringTerm & n) override { - search::query::SimpleRegExpTerm re(vespalib::Regexp::make_from_substring(n.getTerm()), - n.getView(), n.getId(), n.getWeight()); + query::SimpleRegExpTerm re(vespalib::Regexp::make_from_substring(n.getTerm()), + n.getView(), n.getId(), n.getWeight()); visitTerm(re); } void visit(SuffixTerm & n) override { - search::query::SimpleRegExpTerm re(vespalib::Regexp::make_from_suffix(n.getTerm()), - n.getView(), n.getId(), n.getWeight()); + query::SimpleRegExpTerm re(vespalib::Regexp::make_from_suffix(n.getTerm()), + n.getView(), n.getId(), n.getWeight()); visitTerm(re); } void visit(PredicateQuery &n) override { visitPredicate(n); } @@ -529,9 +498,9 @@ public: void createDirectWeightedSet(WS *bp, NODE &n) { Blueprint::UP result(bp); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - vespalib::string term = search::queryeval::termAsString(node); - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); + const query::Node &node = *n.getChildren()[i]; + vespalib::string term = queryeval::termAsString(node); + uint32_t weight = queryeval::getWeightFromNode(node).percent(); bp->addTerm(term, weight); } setResult(std::move(result)); @@ -541,36 +510,34 @@ public: void createShallowWeightedSet(WS *bp, NODE &n, const FieldSpec &fs) { Blueprint::UP result(bp); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); + const query::Node &node = *n.getChildren()[i]; + uint32_t weight = queryeval::getWeightFromNode(node).percent(); const string stack = StackDumpCreator::create(node); FieldSpec childfs = bp->getNextChildField(fs); - bp->addTerm(make_UP(new AttributeFieldBlueprint(childfs, _attr, stack)), weight); + bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, stack), weight); } setResult(std::move(result)); } - void visit(search::query::WeightedSetTerm &n) override { + void visit(query::WeightedSetTerm &n) override { bool isSingleValue = !_attr.hasMultiValue(); bool isString = (_attr.isStringType() && _attr.hasEnum()); bool isInteger = _attr.isIntegerType(); if (isSingleValue && (isString || isInteger)) { - AttributeWeightedSetBlueprint *ws - = new AttributeWeightedSetBlueprint(_field, _attr); - Blueprint::UP result(ws); + auto ws = std::make_unique<AttributeWeightedSetBlueprint>(_field, _attr); for (size_t i = 0; i < n.getChildren().size(); ++i) { - const search::query::Node &node = *n.getChildren()[i]; - uint32_t weight = search::queryeval::getWeightFromNode(node).percent(); - vespalib::string term = search::queryeval::termAsString(node); - search::QueryTermSimple::UP qt; + const query::Node &node = *n.getChildren()[i]; + uint32_t weight = queryeval::getWeightFromNode(node).percent(); + vespalib::string term = queryeval::termAsString(node); + QueryTermSimple::UP qt; if (isInteger) { - qt.reset(new search::QueryTermSimple(term, search::QueryTermSimple::WORD)); + qt = std::make_unique<QueryTermSimple>(term, QueryTermSimple::WORD); } else { - qt.reset(new search::QueryTermBase(term, search::QueryTermSimple::WORD)); + qt = std::make_unique<QueryTermBase>(term, QueryTermSimple::WORD); } ws->addToken(_attr.createSearchContext(std::move(qt), attribute::SearchContextParams()), weight); } - setResult(std::move(result)); + setResult(std::move(ws)); } else { if (_dwa != nullptr) { auto *bp = new DirectWeightedSetBlueprint<queryeval::WeightedSetTermSearch>(_field, *_dwa, n.getChildren().size()); @@ -582,7 +549,7 @@ public: } } - void visit(search::query::DotProduct &n) override { + void visit(query::DotProduct &n) override { if (_dwa != nullptr) { auto *bp = new DirectWeightedSetBlueprint<queryeval::DotProductSearch>(_field, *_dwa, n.getChildren().size()); createDirectWeightedSet(bp, n); @@ -592,7 +559,7 @@ public: } } - void visit(search::query::WandTerm &n) override { + void visit(query::WandTerm &n) override { if (_dwa != nullptr) { auto *bp = new DirectWandBlueprint(_field, *_dwa, n.getTargetNumHits(), n.getScoreThreshold(), n.getThresholdBoostFactor(), @@ -614,8 +581,8 @@ public: Blueprint::UP AttributeBlueprintFactory::createBlueprint(const IRequestContext & requestContext, - const FieldSpec &field, - const search::query::Node &term) + const FieldSpec &field, + const query::Node &term) { const IAttributeVector *attr(requestContext.getAttribute(field.getName())); if (attr == nullptr) { @@ -623,8 +590,7 @@ AttributeBlueprintFactory::createBlueprint(const IRequestContext & requestContex } CreateBlueprintVisitor visitor(*this, requestContext, field, *attr); const_cast<Node &>(term).accept(visitor); - Blueprint::UP bp = visitor.getResult(); - return bp; + return visitor.getResult(); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index febf7d101a9..a524f2ce0fa 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -82,7 +82,9 @@ protected: public: AttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData); - bool seekFast(uint32_t docId) const { return _searchContext.cmp(docId); } + bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); } + + const attribute::ISearchContext * getAttributeSearchContext() const override { return &_searchContext; } }; template <typename SC> @@ -100,7 +102,7 @@ protected: public: FilterAttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData); - bool seekFast(uint32_t docId) const { return _searchContext.cmp(docId); } + bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index fb47ad0cfcc..9b1e837beac 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -16,14 +16,14 @@ namespace search { template <typename SC> void AttributeIteratorBase::and_hits_into(const SC & sc, BitVector & result, uint32_t begin_id) const { - result.foreach_truebit([&](uint32_t key) { if ( ! sc.cmp(key)) { result.clearBit(key); }}, begin_id); + result.foreach_truebit([&](uint32_t key) { if ( ! sc.matches(key)) { result.clearBit(key); }}, begin_id); result.invalidateCachedCount(); } template <typename SC> void AttributeIteratorBase::or_hits_into(const SC & sc, BitVector & result, uint32_t begin_id) const { - result.foreach_falsebit([&](uint32_t key) { if ( sc.cmp(key)) { result.setBit(key); }}, begin_id); + result.foreach_falsebit([&](uint32_t key) { if ( sc.matches(key)) { result.setBit(key); }}, begin_id); result.invalidateCachedCount(); } @@ -33,7 +33,7 @@ std::unique_ptr<BitVector> AttributeIteratorBase::get_hits(const SC & sc, uint32_t begin_id) const { BitVector::UP result = BitVector::create(begin_id, getEndId()); for (uint32_t docId(std::max(begin_id, getDocId())); docId < getEndId(); docId++) { - if (sc.cmp(docId)) { + if (sc.matches(docId)) { result->setBit(docId); } } @@ -338,7 +338,7 @@ AttributeIteratorT<SC>::doSeek(uint32_t docId) { if (isAtEnd(docId)) { setAtEnd(); - } else if (_searchContext.cmp(docId, _weight)) { + } else if (_searchContext.matches(docId, _weight)) { setDocId(docId); } } @@ -349,7 +349,7 @@ FilterAttributeIteratorT<SC>::doSeek(uint32_t docId) { if (isAtEnd(docId)) { setAtEnd(); - } else if (_searchContext.cmp(docId)) { + } else if (_searchContext.matches(docId)) { setDocId(docId); } } @@ -359,7 +359,7 @@ void AttributeIteratorStrict<SC>::doSeek(uint32_t docId) { for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) { - if (_searchContext.cmp(nextId, _weight)) { + if (_searchContext.matches(nextId, _weight)) { setDocId(nextId); return; } @@ -372,7 +372,7 @@ void FilterAttributeIteratorStrict<SC>::doSeek(uint32_t docId) { for (uint32_t nextId = docId; !isAtEnd(nextId); ++nextId) { - if (_searchContext.cmp(nextId)) { + if (_searchContext.matches(nextId)) { setDocId(nextId); return; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 5261eb0bb05..1ccc9923c99 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -73,7 +73,7 @@ AttributeVector::BaseName::BaseName(const vespalib::stringref &base, append(name); } -AttributeVector::BaseName::~BaseName() { } +AttributeVector::BaseName::~BaseName() = default; AttributeVector::BaseName::string @@ -165,9 +165,7 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _genHandler(), _genHolder(), _status(Status::createName((_baseFileName.getIndexName() + - (_baseFileName.getSnapshotName().empty() ? - "" : - ".") + + (_baseFileName.getSnapshotName().empty() ? "" : ".") + _baseFileName.getSnapshotName()), _baseFileName.getAttributeName())), _highestValueCount(1), @@ -177,29 +175,24 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _createSerialNum(0u), _compactLidSpaceGeneration(0u), _hasEnum(false), - _hasSortedEnum(false), _loaded(false), _enableEnumeratedSave(false) { } - -AttributeVector::~AttributeVector() { } +AttributeVector::~AttributeVector() = default; void AttributeVector::updateStat(bool force) { if (force) { onUpdateStat(); } else if (_nextStatUpdateTime < fastos::ClockSystem::now()) { onUpdateStat(); - _nextStatUpdateTime = fastos::ClockSystem::now() + - fastos::TimeStamp::SEC; + _nextStatUpdateTime = fastos::ClockSystem::now() + fastos::TimeStamp::SEC; } } -size_t AttributeVector::getFixedWidth() const { return _config.basicType().fixedSize(); } bool AttributeVector::hasEnum() const { return _hasEnum; } bool AttributeVector::hasEnum2Value() const { return false; } uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount; } -uint32_t AttributeVector::getNumDocs() const { return _status.getNumDocs(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) @@ -217,13 +210,11 @@ AttributeVector::commit(bool forceUpdateStat) _loaded = true; } - void AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) { if (firstSyncToken < getStatus().getLastSyncToken()) { - LOG(error, - "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", + LOG(error, "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", getStatus().getLastSyncToken(), firstSyncToken); abort(); } @@ -231,7 +222,6 @@ AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) _status.setLastSyncToken(lastSyncToken); } - bool AttributeVector::addDocs(DocId &startDoc, DocId &lastDoc, uint32_t numDocs) { @@ -271,19 +261,10 @@ AttributeVector::incGeneration() void -AttributeVector::updateStatistics(uint64_t numValues, - uint64_t numUniqueValue, - uint64_t allocated, - uint64_t used, - uint64_t dead, - uint64_t onHold) +AttributeVector::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, + uint64_t used, uint64_t dead, uint64_t onHold) { - _status.updateStatistics(numValues, - numUniqueValue, - allocated, - used, - dead, - onHold); + _status.updateStatistics(numValues, numUniqueValue, allocated, used, dead, onHold); } AddressSpace @@ -292,16 +273,6 @@ AttributeVector::getEnumStoreAddressSpaceUsage() const return AddressSpaceUsage::defaultEnumStoreUsage(); } -bool -AttributeVector::hasMultiValue() const { - return _config.collectionType().isMultiValue(); -} - -bool -AttributeVector::hasWeightedSetType() const { - return _config.collectionType().isWeightedSet(); -} - AddressSpace AttributeVector::getMultiValueAddressSpaceUsage() const { @@ -311,38 +282,7 @@ AttributeVector::getMultiValueAddressSpaceUsage() const AddressSpaceUsage AttributeVector::getAddressSpaceUsage() const { - return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), - getMultiValueAddressSpaceUsage()); -} - -const vespalib::string & -AttributeVector::getName() const { - return _baseFileName.getAttributeName(); -} - -attribute::BasicType::Type -AttributeVector::getBasicType() const { - return getInternalBasicType().type(); -} -attribute::CollectionType::Type -AttributeVector::getCollectionType() const { - return getInternalCollectionType().type(); -} - -bool -AttributeVector::getIsFilter() const { - return _config.getIsFilter(); -} - -bool -AttributeVector::getIsFastSearch() const { - return _config.fastSearch(); -} - -uint32_t -AttributeVector::getCommittedDocIdLimit() const -{ - return _committedDocIdLimit; + return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage()); } bool @@ -585,7 +525,7 @@ AttributeVector::createSearchContext(QueryTermSimpleUP term, return getSearch(std::move(term), params); } -AttributeVector::SearchContext::~SearchContext() { } +AttributeVector::SearchContext::~SearchContext() = default; unsigned int AttributeVector::SearchContext::approximateHits() const diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index b25c7b67299..87ef9a41432 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -213,7 +213,6 @@ protected: void setEnumMax(uint32_t e) { _enumMax = e; setEnum(); } void setEnum(bool hasEnum_=true) { _hasEnum = hasEnum_; } - void setSortedEnum(bool sorted=true) { _hasSortedEnum = sorted; } void setNumDocs(uint32_t n) { _status.setNumDocs(n); } void incNumDocs() { _status.incNumDocs(); } @@ -398,7 +397,7 @@ public: bool isLoaded() const { return _loaded; } /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ - virtual size_t getFixedWidth() const override; + size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } @@ -406,17 +405,16 @@ public: void setBaseFileName(const vespalib::stringref & name) { _baseFileName = name; } // Implements IAttributeVector - virtual const vespalib::string & getName() const override; + const vespalib::string & getName() const override final { return _baseFileName.getAttributeName(); } bool hasArrayType() const { return _config.collectionType().isArray(); } - bool hasEnum() const override; - bool hasSortedEnum() const { return _hasSortedEnum; } + bool hasEnum() const override final; virtual bool hasEnum2Value() const; uint32_t getMaxValueCount() const override; uint32_t getEnumMax() const { return _enumMax; } // Implements IAttributeVector - uint32_t getNumDocs() const override; + uint32_t getNumDocs() const override final { return _status.getNumDocs(); } uint32_t & getCommittedDocIdLimitRef() { return _committedDocIdLimit; } void setCommittedDocIdLimit(uint32_t committedDocIdLimit) { _committedDocIdLimit = committedDocIdLimit; @@ -427,13 +425,12 @@ public: AddressSpaceUsage getAddressSpaceUsage() const; - // Implements IAttributeVector - virtual BasicType::Type getBasicType() const override; - virtual CollectionType::Type getCollectionType() const override; - virtual bool getIsFilter() const override; - virtual bool getIsFastSearch() const override; - virtual uint32_t getCommittedDocIdLimit() const override; - virtual bool isImported() const override; + BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); } + CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); } + bool getIsFilter() const override final { return _config.getIsFilter(); } + bool getIsFastSearch() const override final { return _config.fastSearch(); } + uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit; } + bool isImported() const override; /** * Updates the base file name of this attribute vector and saves @@ -604,7 +601,6 @@ private: uint64_t _createSerialNum; uint64_t _compactLidSpaceGeneration; bool _hasEnum; - bool _hasSortedEnum; bool _loaded; bool _enableEnumeratedSave; fastos::TimeStamp _nextStatUpdateTime; @@ -634,8 +630,8 @@ private: friend class AttributeManagerTest; public: bool headerTypeOK(const vespalib::GenericHeader &header) const; - bool hasMultiValue() const override; - bool hasWeightedSetType() const override; + bool hasMultiValue() const override final { return _config.collectionType().isMultiValue(); } + bool hasWeightedSetType() const override final { return _config.collectionType().isWeightedSet(); } /** * Should be called by the writer thread. */ diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index abcdc0244c2..8f67d237a60 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -170,7 +170,6 @@ StringDirectAttrVector(const vespalib::string & baseFileName, const Config & c) _idx.push_back(0); } setEnum(); - setSortedEnum(true); } template <typename F> @@ -182,6 +181,5 @@ StringDirectAttrVector(const vespalib::string & baseFileName) : _idx.push_back(0); } setEnum(); - setSortedEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp b/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp new file mode 100644 index 00000000000..3be4c478376 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp @@ -0,0 +1,47 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "elementiterator.h" +#include <vespa/searchcommon/attribute/i_search_context.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> + +using search::fef::TermFieldMatchDataPosition; + +namespace search::attribute { + +void +ElementIterator::doSeek(uint32_t docid) { + _search->doSeek(docid); + setDocId(_search->getDocId()); +} + +void +ElementIterator::doUnpack(uint32_t docid) { + _tfmd.reset(docid); + int32_t weight(0); + for (int32_t id = _searchContext.find(docid, 0, weight); id >= 0; id = _searchContext.find(docid, id+1, weight)) { + _tfmd.appendPosition(TermFieldMatchDataPosition(id, 0, weight, 1)); + } +} + +vespalib::Trinary +ElementIterator::is_strict() const { + return _search->is_strict(); +} + +void +ElementIterator::initRange(uint32_t beginid, uint32_t endid) { + SearchIterator::initRange(beginid, endid); + _search->initRange(beginid, endid); + setDocId(_search->getDocId()); +} + +ElementIterator::ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd) + : _search(std::move(search)), + _searchContext(sc), + _tfmd(tfmd) +{ +} + +ElementIterator::~ElementIterator() = default; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.h b/searchlib/src/vespa/searchlib/attribute/elementiterator.h new file mode 100644 index 00000000000..232139751b6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/elementiterator.h @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/queryeval/searchiterator.h> + +namespace search::fef { class TermFieldMatchData; } +namespace search::attribute { + +class ISearchContext; + +class ElementIterator : public queryeval::SearchIterator +{ +private: + SearchIterator::UP _search; + const ISearchContext & _searchContext; + fef::TermFieldMatchData & _tfmd; + + void doSeek(uint32_t docid) override; + void doUnpack(uint32_t docid) override; + Trinary is_strict() const override; + void initRange(uint32_t beginid, uint32_t endid) override; +public: + ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd); + ~ElementIterator(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index d4fa47eac49..ee984688594 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -12,7 +12,6 @@ SingleStringExtAttribute::SingleStringExtAttribute(const vespalib::string & name StringDirectAttrVector< AttrVector::Features<false> >(name, Config(BasicType::STRING, CollectionType::SINGLE)) { setEnum(false); - setSortedEnum(false); } bool SingleStringExtAttribute::addDoc(DocId & docId) @@ -45,7 +44,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name, (name, Config(BasicType::STRING, ctype)) { setEnum(false); - setSortedEnum(false); } MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) : @@ -53,7 +51,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) (name, Config(BasicType::STRING, CollectionType::ARRAY)) { setEnum(false); - setSortedEnum(false); } bool MultiStringExtAttribute::addDoc(DocId & docId) @@ -138,7 +135,6 @@ WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::str WeightedSetExtAttributeBase<MultiStringExtAttribute>(name) { setEnum(false); - setSortedEnum(false); } WeightedSetStringExtAttribute::~WeightedSetStringExtAttribute() {} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index bfcd7f29f29..c480eec5e88 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -44,8 +44,7 @@ ImportedSearchContext::ImportedSearchContext( } -ImportedSearchContext::~ImportedSearchContext() { -} +ImportedSearchContext::~ImportedSearchContext() = default; unsigned int ImportedSearchContext::approximateHits() const { return _reference_attribute.getNumDocs(); diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index f8b434d4c6c..cfb5371dbb9 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -50,7 +50,7 @@ public: const SearchContextParams& params, const ImportedAttributeVector& imported_attribute, const attribute::IAttributeVector &target_attribute); - ~ImportedSearchContext(); + ~ImportedSearchContext() override; std::unique_ptr<queryeval::SearchIterator> @@ -64,16 +64,16 @@ public: using DocId = IAttributeVector::DocId; - bool cmp(DocId docId, int32_t& weight) const { - return _target_search_context->cmp(getTargetLid(docId), weight); + int32_t find(DocId docId, int32_t elemId, int32_t& weight) const { + return _target_search_context->find(getTargetLid(docId), elemId, weight); } - bool cmp(DocId docId) const { - return _target_search_context->cmp(getTargetLid(docId)); + int32_t find(DocId docId, int32_t elemId) const { + return _target_search_context->find(getTargetLid(docId), elemId); } - bool onCmp(uint32_t docId, int32_t &weight) const override { return cmp(docId, weight); } - bool onCmp(uint32_t docId) const override { return cmp(docId); } + int32_t onFind(uint32_t docId, int32_t elemId, int32_t &weight) const override { return find(docId, elemId, weight); } + int32_t onFind(uint32_t docId, int32_t elemId) const override { return find(docId, elemId); } const ReferenceAttribute& attribute() const noexcept { return _reference_attribute; } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h index 1e02e4d39b0..9d0cac64adf 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h @@ -71,12 +71,12 @@ public: private: const MultiValueNumericAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override; @@ -86,25 +86,25 @@ public: Int64Range getAsIntegerTerm() const override; - bool cmp(DocId doc, int32_t & weight) const { + int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - weight = mv.weight(); - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + weight = values[i].weight(); + return i; } } - return false; + return -1; } - bool cmp(DocId doc) const { + int32_t find(DocId doc, int32_t elemId) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + return i; } } - return false; + return -1; } std::unique_ptr<queryeval::SearchIterator> @@ -119,12 +119,12 @@ public: private: const MultiValueNumericAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } protected: @@ -132,27 +132,27 @@ public: public: ArraySearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId doc, int32_t & weight) const { - uint32_t hitCount = 0; + int32_t find(DocId doc, int32_t elemId, int32_t & weight) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - hitCount++; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + weight = 1; + return i; } } - weight = hitCount; + weight = 0; - return hitCount != 0; + return -1; } - bool cmp(DocId doc) const { + int32_t find(DocId doc, int32_t elemId) const { MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); - for (const MultiValueType &mv : values) { - if (this->match(mv.value())) { - return true; + for (uint32_t i(elemId); i < values.size(); i++) { + if (this->match(values[i].value())) { + return i; } } - return false; + return -1; } Int64Range getAsIntegerTerm() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 66dbbfeb1da..b25438f8f2a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -52,12 +52,12 @@ protected: protected: const MultiValueNumericEnumAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override { return this->isValid(); } @@ -65,31 +65,31 @@ protected: public: SetSearchContext(QueryTermSimpleUP qTerm, const NumericAttribute & toBeSearched); - bool - cmp(DocId doc, int32_t & weight) const + int32_t + find(DocId doc, int32_t elemId, int32_t & weight) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - weight = wi.weight(); - return true; + weight = indices[i].weight(); + return i; } } - return false; + return -1; } - bool - cmp(DocId doc) const + int32_t + find(DocId doc, int32_t elemId) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - return true; + return i; } } - return false; + return -1; } Int64Range getAsIntegerTerm() const override; @@ -105,12 +105,12 @@ protected: protected: const MultiValueNumericEnumAttribute<B, M> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override { return this->isValid(); } @@ -119,34 +119,34 @@ protected: ArraySearchContext(QueryTermSimpleUP qTerm, const NumericAttribute & toBeSearched); Int64Range getAsIntegerTerm() const override; - bool - cmp(DocId doc, int32_t & weight) const + int32_t + find(DocId doc, int32_t elemId, int32_t & weight) const { - uint32_t hitCount = 0; WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - hitCount++; + weight = 1; + return i; } } - weight = hitCount; + weight = 0; - return hitCount != 0; + return -1; } bool - cmp(DocId doc) const + find(DocId doc, int32_t elemId) const { WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - T v = _toBeSearched._enumStore.getValue(wi.value()); + for (uint32_t i(elemId); i < indices.size(); i++) { + T v = _toBeSearched._enumStore.getValue(indices[i].value()); if (this->match(v)) { - return true; + return i; } } - return false; + return -1; } std::unique_ptr<queryeval::SearchIterator> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 7edbcedcb2e..5e5d33419aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -119,10 +119,10 @@ public: const MultiValueStringAttributeT<B, M> & myAttribute() const { return static_cast< const MultiValueStringAttributeT<B, M> & > (attribute()); } - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elemId) const override; template <typename Collector> - bool collectWeight(DocId doc, int32_t & weight, Collector & collector) const; + int32_t findNextWeight(DocId doc, int32_t elemId, int32_t & weight, Collector & collector) const; }; /* @@ -134,7 +134,7 @@ public: StringImplSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId docId, int32_t & weight) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; }; /* @@ -146,7 +146,7 @@ public: StringImplSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId docId, int32_t & weight) const override; + int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; }; template <typename BT> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index a68ac218784..858fe579764 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -62,47 +62,47 @@ private: } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringSetImplSearchContext::onCmp(DocId doc, int32_t & weight) const +int32_t +MultiValueStringAttributeT<B, M>::StringSetImplSearchContext::onFind(DocId doc, int32_t elemId, int32_t &weight) const { StringAttribute::StringSearchContext::CollectWeight collector; - return this->collectWeight(doc, weight, collector); + return this->findNextWeight(doc, elemId, weight, collector); } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringArrayImplSearchContext::onCmp(DocId doc, int32_t & weight) const +int32_t +MultiValueStringAttributeT<B, M>::StringArrayImplSearchContext::onFind(DocId doc, int32_t elemId, int32_t &weight) const { StringAttribute::StringSearchContext::CollectHitCount collector; - return this->collectWeight(doc, weight, collector); + return this->findNextWeight(doc, elemId, weight, collector); } template <typename B, typename M> template <typename Collector> -bool -MultiValueStringAttributeT<B, M>::StringImplSearchContext::collectWeight(DocId doc, int32_t & weight, Collector & collector) const +int32_t +MultiValueStringAttributeT<B, M>::StringImplSearchContext::findNextWeight(DocId doc, int32_t elemId, int32_t & weight, Collector & collector) const { WeightedIndexArrayRef indices(myAttribute()._mvMapping.get(doc)); EnumAccessor<typename B::EnumStore> accessor(myAttribute()._enumStore); - collectMatches(indices, accessor, collector); + int32_t foundElem = findNextMatch(indices, elemId, accessor, collector); weight = collector.getWeight(); - return collector.hasMatch(); + return foundElem; } template <typename B, typename M> -bool -MultiValueStringAttributeT<B, M>::StringImplSearchContext::onCmp(DocId doc) const +int32_t +MultiValueStringAttributeT<B, M>::StringImplSearchContext::onFind(DocId doc, int32_t elemId) const { const MultiValueStringAttributeT<B, M> & attr(static_cast< const MultiValueStringAttributeT<B, M> & > (attribute())); WeightedIndexArrayRef indices(attr._mvMapping.get(doc)); - for (const WeightedIndex &wi : indices) { - if (isMatch(attr._enumStore.getValue(wi.value()))) { - return true; + for (uint32_t i(elemId); i < indices.size(); i++) { + if (isMatch(attr._enumStore.getValue(indices[i].value()))) { + return i; } } - return false; + return -1; } template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 48f9d0404c5..7793af8f510 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -113,7 +113,8 @@ template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(bool strict) { - assert(!_fetchPostingsDone); + assert (! _fetchPostingsDone); + _fetchPostingsDone = true; if (_uniqueValues < 2u) { return; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h index 8edbf6cde59..5bdcc3dfe63 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h @@ -41,27 +41,29 @@ private: private: const T * _data; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int elemId) const override { + return find(docId, elemId); } bool valid() const override; public: SingleSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; const T v = _data[docId]; weight = 1; - return this->match(v); + return this->match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int elemId) const { + if ( elemId != 0) return -1; const T v = _data[docId]; - return this->match(v); + return this->match(v) ? 0 : -1; } Int64Range getAsIntegerTerm() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index 7dca10e83e4..096c159115e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -58,12 +58,12 @@ protected: protected: const SingleValueNumericEnumAttribute<B> & _toBeSearched; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override { + return find(docId, elemId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elemId) const override { + return find(docId, elemId); } bool valid() const override; @@ -72,15 +72,17 @@ protected: Int64Range getAsIntegerTerm() const override; - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; T v = _toBeSearched._enumStore.getValue(_toBeSearched.getEnumIndex(docId)); weight = 1; - return this->match(v); + return this->match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int32_t elemId) const { + if ( elemId != 0) return -1; T v = _toBeSearched._enumStore.getValue(_toBeSearched.getEnumIndex(docId)); - return this->match(v); + return this->match(v) ? 0 : -1; } std::unique_ptr<queryeval::SearchIterator> diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h index 9320e248160..f5f666bd89f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h @@ -67,12 +67,12 @@ public: uint32_t _valueShiftMask; uint32_t _wordShift; - bool onCmp(DocId docId, int32_t & weight) const override { - return cmp(docId, weight); + int32_t onFind(DocId docId, int32_t elementId, int32_t & weight) const override { + return find(docId, elementId, weight); } - bool onCmp(DocId docId) const override { - return cmp(docId); + int32_t onFind(DocId docId, int32_t elementId) const override { + return find(docId, elementId); } bool valid() const override; @@ -80,19 +80,21 @@ public: public: SingleSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const NumericAttribute & toBeSearched); - bool cmp(DocId docId, int32_t & weight) const { + int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { + if ( elemId != 0) return -1; const Word &word = _wordData[docId >> _wordShift]; uint32_t valueShift = (docId & _valueShiftMask) << _valueShiftShift; T v = (word >> valueShift) & _valueMask; weight = 1; - return match(v); + return match(v) ? 0 : -1; } - bool cmp(DocId docId) const { + int32_t find(DocId docId, int32_t elemId) const { + if ( elemId != 0) return -1; const Word &word = _wordData[docId >> _wordShift]; uint32_t valueShift = (docId & _valueShiftMask) << _valueShiftShift; T v = (word >> valueShift) & _valueMask; - return match(v); + return match(v) ? 0 : -1; } Int64Range getAsIntegerTerm() const override; @@ -101,14 +103,10 @@ public: createFilterIterator(fef::TermFieldMatchData * matchData, bool strict) override; }; - SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, - const Config &c, - Word valueMask, - uint32_t valueShiftShift, - uint32_t valueShiftMask, - uint32_t wordShift); + SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, const Config &c, Word valueMask, + uint32_t valueShiftShift, uint32_t valueShiftMask, uint32_t wordShift); - ~SingleValueSmallNumericAttribute(); + ~SingleValueSmallNumericAttribute() override; uint32_t getValueCount(DocId doc) const override { if (doc >= B::getNumDocs()) { @@ -125,7 +123,8 @@ public: bool onLoad() override; void onSave(IAttributeSaveTarget &saveTarget) override; - SearchContext::UP getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; + SearchContext::UP + getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; T getFast(DocId doc) const { const Word &word = _wordData[doc >> _wordShift]; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h index 227c1d0667c..4993b295b37 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h @@ -89,14 +89,15 @@ public: StringSearchContext(std::move(qTerm), toBeSearched) { } protected: - bool onCmp(DocId doc, int32_t & weight) const override { + int32_t onFind(DocId doc, int32_t elemId, int32_t &weight) const override { weight = 1; - return onCmp(doc); + return onFind(doc, elemId); } - bool onCmp(DocId doc) const override { + int32_t onFind(DocId doc, int32_t elemId) const override { + if ( elemId != 0) return -1; const SingleValueStringAttributeT<B> & attr(static_cast<const SingleValueStringAttributeT<B> &>(attribute())); - return isMatch(attr._enumStore.getValue(attr._enumIndices[doc])); + return isMatch(attr._enumStore.getValue(attr._enumIndices[doc])) ? 0 : -1; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 5ba936b5f52..16a05e5f0a9 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -273,31 +273,31 @@ public: } -bool -StringAttribute::StringSearchContext::onCmp(DocId docId, int32_t & weight) const +int32_t +StringAttribute::StringSearchContext::onFind(DocId docId, int32_t elemId, int32_t &weight) const { WeightedConstChar * buffer = getBuffer(); uint32_t valueCount = attribute().get(docId, buffer, _bufferLen); CollectWeight collector; DirectAccessor accessor; - collectMatches(vespalib::ConstArrayRef<WeightedConstChar>(buffer, std::min(valueCount, _bufferLen)), accessor, collector); + int32_t foundElem = findNextMatch(vespalib::ConstArrayRef<WeightedConstChar>(buffer, std::min(valueCount, _bufferLen)), elemId, accessor, collector); weight = collector.getWeight(); - return collector.hasMatch(); + return foundElem; } -bool -StringAttribute::StringSearchContext::onCmp(DocId docId) const +int32_t +StringAttribute::StringSearchContext::onFind(DocId docId, int32_t elemId) const { WeightedConstChar * buffer = getBuffer(); uint32_t valueCount = attribute().get(docId, buffer, _bufferLen); - for (uint32_t i = 0, m = std::min(valueCount, _bufferLen); (i < m); i++) { + for (uint32_t i = elemId, m = std::min(valueCount, _bufferLen); (i < m); i++) { if (isMatch(buffer[i].getValue())) { - return true; + return i; } } - return false; + return -1; } bool StringAttribute::applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index 5e0847f3039..c817332af15 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -96,7 +96,7 @@ private: class StringSearchContext : public SearchContext { public: StringSearchContext(QueryTermSimpleUP qTerm, const StringAttribute & toBeSearched); - virtual ~StringSearchContext(); + ~StringSearchContext() override; private: bool _isPrefix; bool _isRegex; @@ -147,17 +147,19 @@ private: }; template<typename WeightedT, typename Accessor, typename Collector> - void collectMatches(vespalib::ConstArrayRef<WeightedT> w, const Accessor & ac, Collector & collector) const { - for (const WeightedT &wRef : w) { - if (isMatch(ac.get(wRef.value()))) { - collector.addWeight(wRef.weight()); + int32_t findNextMatch(vespalib::ConstArrayRef<WeightedT> w, int32_t elemId, const Accessor & ac, Collector & collector) const { + for (uint32_t i(elemId); i < w.size(); i++) { + if (isMatch(ac.get(w[i].value()))) { + collector.addWeight(w[i].weight()); + return i; } } + return -1; } - bool onCmp(DocId docId, int32_t & weight) const override; - bool onCmp(DocId docId) const override; + int32_t onFind(DocId docId, int32_t elementId, int32_t &weight) const override; + int32_t onFind(DocId docId, int32_t elementId) const override; bool isPrefix() const { return _isPrefix; } bool isRegex() const { return _isRegex; } @@ -166,7 +168,7 @@ private: const vespalib::Regexp * getRegex() const { return _regex.get(); } private: WeightedConstChar * getBuffer() const { - if (_buffer == NULL) { + if (_buffer == nullptr) { _buffer = new WeightedConstChar[_bufferLen]; } return _buffer; diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.cpp b/searchlib/src/vespa/searchlib/common/locationiterators.cpp index e460729e663..16e465bcd05 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.cpp +++ b/searchlib/src/vespa/searchlib/common/locationiterators.cpp @@ -20,7 +20,7 @@ private: public: FastS_2DZLocationIterator(unsigned int numDocs, bool strict, const Location & location); - ~FastS_2DZLocationIterator(); + ~FastS_2DZLocationIterator() override; }; @@ -39,7 +39,7 @@ FastS_2DZLocationIterator(unsigned int numDocs, }; -FastS_2DZLocationIterator::~FastS_2DZLocationIterator() {} +FastS_2DZLocationIterator::~FastS_2DZLocationIterator() = default; void @@ -103,10 +103,8 @@ FastS_2DZLocationIterator::doUnpack(uint32_t docId) } -search::queryeval::SearchIterator * -FastS_AllocLocationIterator(unsigned int numDocs, - bool strict, - const Location & location) +std::unique_ptr<search::queryeval::SearchIterator> +FastS_AllocLocationIterator(unsigned int numDocs, bool strict, const Location & location) { - return new FastS_2DZLocationIterator(numDocs, strict, location); + return std::make_unique<FastS_2DZLocationIterator>(numDocs, strict, location); } diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.h b/searchlib/src/vespa/searchlib/common/locationiterators.h index b913305873f..e345bcae4fe 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.h +++ b/searchlib/src/vespa/searchlib/common/locationiterators.h @@ -2,10 +2,10 @@ #pragma once -#include <vespa/searchlib/queryeval/iterators.h> +#include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/common/location.h> -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FastS_AllocLocationIterator(unsigned int numDocs, bool strict, const search::common::Location & location); diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index ed86014f7b3..e39f11f56b2 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -459,16 +459,16 @@ public: default: case 4: r |= _data[a._idx + a._pos + 3] << 0; - //@fallthrough@ + [[fallthrough]]; case 3: r |= _data[a._idx + a._pos + 2] << 8; - //@fallthrough@ + [[fallthrough]]; case 2: r |= _data[a._idx + a._pos + 1] << 16; - //@fallthrough@ + [[fallthrough]]; case 1: r |= _data[a._idx + a._pos + 0] << 24; - //@fallthrough@ + [[fallthrough]]; case 0:; } a._pos += std::min(4u, left); diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp index 4f4491327e7..834f5913af9 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp @@ -14,8 +14,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.distancetopathfeature"); -namespace search { -namespace features { +namespace search::features { const feature_t DistanceToPathExecutor::DEFAULT_DISTANCE(6400000000.0); @@ -96,9 +95,7 @@ DistanceToPathBlueprint::DistanceToPathBlueprint() : { } -DistanceToPathBlueprint::~DistanceToPathBlueprint() -{ -} +DistanceToPathBlueprint::~DistanceToPathBlueprint() = default; void DistanceToPathBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, @@ -175,5 +172,4 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en return stash.create<DistanceToPathExecutor>(path, pos); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.h b/searchlib/src/vespa/searchlib/features/distancetopathfeature.h index cf268c59a2e..442f1ba3b28 100644 --- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.h +++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/common/feature.h> -namespace search { -namespace features { +namespace search::features { /** * Define the point type that makes up the end-points in our path. @@ -52,7 +51,7 @@ private: public: DistanceToPathBlueprint(); - ~DistanceToPathBlueprint(); + ~DistanceToPathBlueprint() override; void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &visitor) const override; fef::Blueprint::UP createInstance() const override; fef::ParameterDescriptions getDescriptions() const override { @@ -64,6 +63,4 @@ public: }; -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp index b189331878c..cba69ea474a 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp @@ -24,9 +24,8 @@ namespace search { ParseItem::ParseItem(ItemType type, int arity) : PARSEITEM_DEFAULT_CONSTRUCTOR_LIST { - assert(type==ITEM_OR || type==ITEM_WEAK_AND || type==ITEM_EQUIV || - type==ITEM_AND || type==ITEM_NOT || type==ITEM_RANK || - type==ITEM_PHRASE || type==ITEM_ANY || type==ITEM_NEAR || type==ITEM_ONEAR); + assert(type==ITEM_OR || type==ITEM_WEAK_AND || type==ITEM_EQUIV || type==ITEM_AND || type==ITEM_NOT + || type==ITEM_RANK || type==ITEM_ANY || type==ITEM_NEAR || type==ITEM_ONEAR); SetType(type); _arity = arity; } @@ -34,7 +33,8 @@ ParseItem::ParseItem(ItemType type, int arity) ParseItem::ParseItem(ItemType type, int arity, const char *idx) : PARSEITEM_DEFAULT_CONSTRUCTOR_LIST { - assert(type == ITEM_PHRASE || type==ITEM_WEIGHTED_SET || type==ITEM_DOT_PRODUCT || type==ITEM_WAND); + assert(type==ITEM_PHRASE || type==ITEM_SAME_ELEMENT || type==ITEM_WEIGHTED_SET + || type==ITEM_DOT_PRODUCT || type==ITEM_WAND); SetType(type); _arity = arity; SetIndex(idx); @@ -108,16 +108,24 @@ ParseItem::AppendBuffer(RawBuf *buf) const case ITEM_ANY: buf->appendCompressedPositiveNumber(_arity); break; - case ITEM_WEAK_AND: case ITEM_NEAR: case ITEM_ONEAR: buf->appendCompressedPositiveNumber(_arity); buf->appendCompressedPositiveNumber(_arg1); - if (Type() == ITEM_WEAK_AND) { - buf->appendCompressedPositiveNumber(indexLen); - if (indexLen != 0) { - buf->append(_indexName.c_str(), indexLen); - } + break; + case ITEM_SAME_ELEMENT: + buf->appendCompressedPositiveNumber(_arity); + buf->appendCompressedPositiveNumber(indexLen); + if (indexLen != 0) { + buf->append(_indexName.c_str(), indexLen); + } + break; + case ITEM_WEAK_AND: + buf->appendCompressedPositiveNumber(_arity); + buf->appendCompressedPositiveNumber(_arg1); + buf->appendCompressedPositiveNumber(indexLen); + if (indexLen != 0) { + buf->append(_indexName.c_str(), indexLen); } break; case ITEM_WEIGHTED_SET: @@ -199,6 +207,9 @@ ParseItem::GetBufferLen() const case ITEM_PHRASE: len += sizeof(uint32_t) * 2 + indexLen; break; + case ITEM_SAME_ELEMENT: + len += sizeof(uint32_t) * 2 + indexLen; + break; case ITEM_WAND: len += sizeof(uint32_t) * 4 + indexLen; break; diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index 3d5a1a5e2bb..d8cbdc696a5 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -52,7 +52,7 @@ public: ITEM_WEIGHTED_SET = 15, ITEM_WEAK_AND = 16, ITEM_EXACTSTRINGTERM = 17, - UNUSED_LEGACY_ITEM_RISE_QUERY = 18, + ITEM_SAME_ELEMENT = 18, ITEM_PURE_WEIGHTED_STRING = 19, ITEM_PURE_WEIGHTED_LONG = 20, ITEM_DOT_PRODUCT = 21, diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp index a7b41476ec6..6f49dc62686 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp @@ -25,12 +25,12 @@ SimpleQueryStack::~SimpleQueryStack() } void -SimpleQueryStack::Push(search::ParseItem *item) +SimpleQueryStack::Push(ParseItem *item) { // Check if query OK for FirstPage _FP_queryOK &= - ( item->Type() != search::ParseItem::ITEM_UNDEF - && item->Type() != search::ParseItem::ITEM_PAREN + ( item->Type() != ParseItem::ITEM_UNDEF + && item->Type() != ParseItem::ITEM_PAREN ); @@ -40,10 +40,10 @@ SimpleQueryStack::Push(search::ParseItem *item) _numItems++; } -search::ParseItem * +ParseItem * SimpleQueryStack::Pop() { - search::ParseItem *item = _stack; + ParseItem *item = _stack; if (_stack != NULL) { _numItems--; _stack = _stack->_next; @@ -53,9 +53,9 @@ SimpleQueryStack::Pop() } void -SimpleQueryStack::AppendBuffer(search::RawBuf *buf) const +SimpleQueryStack::AppendBuffer(RawBuf *buf) const { - for (search::ParseItem *item = _stack; item != NULL; item = item->_next) { + for (ParseItem *item = _stack; item != NULL; item = item->_next) { item->AppendBuffer(buf); } } @@ -66,7 +66,7 @@ SimpleQueryStack::GetBufferLen() const size_t result; result = 0; - for (const search::ParseItem *item = _stack; + for (const ParseItem *item = _stack; item != NULL; item = item->_next) { result += item->GetBufferLen(); } @@ -90,34 +90,35 @@ class ItemName { public: ItemName() { memset(_name, 'X', sizeof(_name)); - _name[search::ParseItem::ITEM_OR] = '|'; - _name[search::ParseItem::ITEM_WEAK_AND] = 'w'; - _name[search::ParseItem::ITEM_EQUIV] = 'E'; - _name[search::ParseItem::ITEM_AND] = '&'; - _name[search::ParseItem::ITEM_NOT] = '-'; - _name[search::ParseItem::ITEM_ANY] = '?'; - _name[search::ParseItem::ITEM_RANK] = '%'; - _name[search::ParseItem::ITEM_NEAR] = 'N'; - _name[search::ParseItem::ITEM_ONEAR] = 'O'; - _name[search::ParseItem::ITEM_NUMTERM] = '#'; - _name[search::ParseItem::ITEM_TERM] = 't'; - _name[search::ParseItem::ITEM_PURE_WEIGHTED_STRING] = 'T'; - _name[search::ParseItem::ITEM_PURE_WEIGHTED_LONG] = 'L'; - _name[search::ParseItem::ITEM_PREFIXTERM] = '*'; - _name[search::ParseItem::ITEM_SUBSTRINGTERM] = 's'; - _name[search::ParseItem::ITEM_EXACTSTRINGTERM] = 'e'; - _name[search::ParseItem::ITEM_SUFFIXTERM] = 'S'; - _name[search::ParseItem::ITEM_PHRASE] = '"'; - _name[search::ParseItem::ITEM_WEIGHTED_SET] = 'W'; - _name[search::ParseItem::ITEM_DOT_PRODUCT] = 'D'; - _name[search::ParseItem::ITEM_WAND] = 'A'; - _name[search::ParseItem::ITEM_PREDICATE_QUERY] = 'P'; - _name[search::ParseItem::ITEM_REGEXP] = '^'; + _name[ParseItem::ITEM_OR] = '|'; + _name[ParseItem::ITEM_WEAK_AND] = 'w'; + _name[ParseItem::ITEM_EQUIV] = 'E'; + _name[ParseItem::ITEM_AND] = '&'; + _name[ParseItem::ITEM_NOT] = '-'; + _name[ParseItem::ITEM_ANY] = '?'; + _name[ParseItem::ITEM_RANK] = '%'; + _name[ParseItem::ITEM_NEAR] = 'N'; + _name[ParseItem::ITEM_ONEAR] = 'O'; + _name[ParseItem::ITEM_NUMTERM] = '#'; + _name[ParseItem::ITEM_TERM] = 't'; + _name[ParseItem::ITEM_PURE_WEIGHTED_STRING] = 'T'; + _name[ParseItem::ITEM_PURE_WEIGHTED_LONG] = 'L'; + _name[ParseItem::ITEM_PREFIXTERM] = '*'; + _name[ParseItem::ITEM_SUBSTRINGTERM] = 's'; + _name[ParseItem::ITEM_EXACTSTRINGTERM] = 'e'; + _name[ParseItem::ITEM_SUFFIXTERM] = 'S'; + _name[ParseItem::ITEM_PHRASE] = '"'; + _name[ParseItem::ITEM_SAME_ELEMENT] = 'M'; + _name[ParseItem::ITEM_WEIGHTED_SET] = 'W'; + _name[ParseItem::ITEM_DOT_PRODUCT] = 'D'; + _name[ParseItem::ITEM_WAND] = 'A'; + _name[ParseItem::ITEM_PREDICATE_QUERY] = 'P'; + _name[ParseItem::ITEM_REGEXP] = '^'; } - char operator[] (search::ParseItem::ItemType i) const { return _name[i]; } + char operator[] (ParseItem::ItemType i) const { return _name[i]; } char operator[] (size_t i) const { return _name[i]; } private: - char _name[search::ParseItem::ITEM_MAX]; + char _name[ParseItem::ITEM_MAX]; }; static ItemName _G_ItemName; @@ -162,29 +163,29 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) while (p < ep) { vespalib::string metaStr; rawtype = *p++; - type = search::ParseItem::GetType(rawtype); - if (search::ParseItem::GetFeature_Weight(rawtype)) { + type = ParseItem::GetType(rawtype); + if (ParseItem::GetFeature_Weight(rawtype)) { int64_t tmpLong(0); p += vespalib::compress::Integer::decompress(tmpLong, p); metaStr.append("(w:"); metaStr.append(make_string("%ld", tmpLong)); metaStr.append(")"); } - if (search::ParseItem::getFeature_UniqueId(rawtype)) { + if (ParseItem::getFeature_UniqueId(rawtype)) { p += vespalib::compress::Integer::decompressPositive(tmp, p); metaStr.append("(u:"); metaStr.append(make_string("%ld", tmp)); metaStr.append(")"); } - if (search::ParseItem::getFeature_Flags(rawtype)) { + if (ParseItem::getFeature_Flags(rawtype)) { flags = *p++; metaStr.append("(f:"); metaStr.append(make_string("%d", flags)); metaStr.append(")"); } - if (search::ParseItem::GetCreator(flags) != search::ParseItem::CREA_ORIG) { + if (ParseItem::GetCreator(flags) != ParseItem::CREA_ORIG) { metaStr.append("(c:"); - metaStr.append(make_string("%d", search::ParseItem::GetCreator(flags))); + metaStr.append(make_string("%d", ParseItem::GetCreator(flags))); metaStr.append(")"); } @@ -192,41 +193,52 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) result.append(metaStr); switch (type) { - case search::ParseItem::ITEM_OR: - case search::ParseItem::ITEM_AND: - case search::ParseItem::ITEM_EQUIV: - case search::ParseItem::ITEM_NOT: - case search::ParseItem::ITEM_RANK: - case search::ParseItem::ITEM_ANY: + case ParseItem::ITEM_OR: + case ParseItem::ITEM_AND: + case ParseItem::ITEM_EQUIV: + case ParseItem::ITEM_NOT: + case ParseItem::ITEM_RANK: + case ParseItem::ITEM_ANY: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; result.append(make_string("%c/%d~", _G_ItemName[type], arity)); break; - case search::ParseItem::ITEM_WEAK_AND: - case search::ParseItem::ITEM_NEAR: - case search::ParseItem::ITEM_ONEAR: + case ParseItem::ITEM_NEAR: + case ParseItem::ITEM_ONEAR: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); arg1 = tmp; - if (type == search::ParseItem::ITEM_WEAK_AND) { - p += vespalib::compress::Integer::decompressPositive(tmp, p); - idxRefLen = tmp; - idxRef = p; - p += idxRefLen; - result.append(make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef)); - } else { - result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1)); - } + result.append(make_string("%c/%d/%d~", _G_ItemName[type], arity, arg1)); + break; + case ParseItem::ITEM_WEAK_AND: + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arity = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arg1 = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + idxRefLen = tmp; + idxRef = p; + p += idxRefLen; + result.append(make_string("%c/%d/%d/%d:%.*s~", _G_ItemName[type], arity, arg1, idxRefLen, idxRefLen, idxRef)); + break; + case ParseItem::ITEM_SAME_ELEMENT: + p += vespalib::compress::Integer::decompressPositive(tmp, p); + arity = tmp; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + idxRefLen = tmp; + idxRef = p; + p += idxRefLen; + result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef)); break; - case search::ParseItem::ITEM_NUMTERM: - case search::ParseItem::ITEM_TERM: - case search::ParseItem::ITEM_PREFIXTERM: - case search::ParseItem::ITEM_SUBSTRINGTERM: - case search::ParseItem::ITEM_EXACTSTRINGTERM: - case search::ParseItem::ITEM_SUFFIXTERM: - case search::ParseItem::ITEM_REGEXP: + case ParseItem::ITEM_NUMTERM: + case ParseItem::ITEM_TERM: + case ParseItem::ITEM_PREFIXTERM: + case ParseItem::ITEM_SUBSTRINGTERM: + case ParseItem::ITEM_EXACTSTRINGTERM: + case ParseItem::ITEM_SUFFIXTERM: + case ParseItem::ITEM_REGEXP: p += vespalib::compress::Integer::decompressPositive(tmp, p); idxRefLen = tmp; idxRef = p; @@ -239,7 +251,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) idxRefLen, idxRefLen, idxRef, termRefLen, termRefLen, termRef)); break; - case search::ParseItem::ITEM_PURE_WEIGHTED_STRING: + case ParseItem::ITEM_PURE_WEIGHTED_STRING: p += vespalib::compress::Integer::decompressPositive(tmp, p); termRefLen = tmp; termRef = p; @@ -248,23 +260,23 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) termRefLen, termRefLen, termRef)); break; - case search::ParseItem::ITEM_PURE_WEIGHTED_LONG: + case ParseItem::ITEM_PURE_WEIGHTED_LONG: tmp = vespalib::nbo::n2h(*reinterpret_cast<const uint64_t *>(p)); p += sizeof(uint64_t); result.append(make_string("%c/%lu", _G_ItemName[type], tmp)); break; - case search::ParseItem::ITEM_PHRASE: - case search::ParseItem::ITEM_WEIGHTED_SET: - case search::ParseItem::ITEM_DOT_PRODUCT: - case search::ParseItem::ITEM_WAND: + case ParseItem::ITEM_PHRASE: + case ParseItem::ITEM_WEIGHTED_SET: + case ParseItem::ITEM_DOT_PRODUCT: + case ParseItem::ITEM_WAND: p += vespalib::compress::Integer::decompressPositive(tmp, p); arity = tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); idxRefLen = tmp; idxRef = p; p += idxRefLen; - if (type == search::ParseItem::ITEM_WAND) { + if (type == ParseItem::ITEM_WAND) { p += vespalib::compress::Integer::decompressPositive(tmp, p); uint32_t targetNumHits = tmp; double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); @@ -279,7 +291,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) } break; - case search::ParseItem::ITEM_PREDICATE_QUERY: + case ParseItem::ITEM_PREDICATE_QUERY: { idxRefLen = static_cast<uint32_t>(ReadCompressedPositiveInt(p)); idxRef = p; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index ec34b2d3a84..542e7990c1e 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -34,9 +34,7 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(const vespalib::strin { } -SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() -{ -} +SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() = default; vespalib::string SimpleQueryStackDumpIterator::readString(const char *&p) { if (p >= _bufEnd) throw false; @@ -154,6 +152,20 @@ SimpleQueryStackDumpIterator::next() _currTerm = NULL; _currTermLen = 0; break; + case ParseItem::ITEM_SAME_ELEMENT: + if (p >= _bufEnd) return false; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + _currArity = tmp; + _currArg1 = 0; + p += vespalib::compress::Integer::decompressPositive(tmp, p); + _currIndexNameLen = tmp; + if (p > _bufEnd) return false; + _currIndexName = p; + p += _currIndexNameLen; + if (p > _bufEnd) return false; + _currTerm = NULL; + _currTermLen = 0; + break; case ParseItem::ITEM_PURE_WEIGHTED_STRING: if (p >= _bufEnd) return false; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h index 938b1bc5542..861a94d1990 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h @@ -75,30 +75,30 @@ struct PredicateHash { // handle the last 23 bytes c += origLen; switch(len) { // all the case statements fall through - case 23: c+=((0xffLL & aKey[offset+22])<<56); //@fallthrough@ - case 22: c+=((0xffLL & aKey[offset+21])<<48); //@fallthrough@ - case 21: c+=((0xffLL & aKey[offset+20])<<40); //@fallthrough@ - case 20: c+=((0xffLL & aKey[offset+19])<<32); //@fallthrough@ - case 19: c+=((0xffLL & aKey[offset+18])<<24); //@fallthrough@ - case 18: c+=((0xffLL & aKey[offset+17])<<16); //@fallthrough@ - case 17: c+=((0xffLL & aKey[offset+16])<<8); //@fallthrough@ + case 23: c+=((0xffLL & aKey[offset+22])<<56); [[fallthrough]]; + case 22: c+=((0xffLL & aKey[offset+21])<<48); [[fallthrough]]; + case 21: c+=((0xffLL & aKey[offset+20])<<40); [[fallthrough]]; + case 20: c+=((0xffLL & aKey[offset+19])<<32); [[fallthrough]]; + case 19: c+=((0xffLL & aKey[offset+18])<<24); [[fallthrough]]; + case 18: c+=((0xffLL & aKey[offset+17])<<16); [[fallthrough]]; + case 17: c+=((0xffLL & aKey[offset+16])<<8); [[fallthrough]]; // the first byte of c is reserved for the length - case 16: b+=((0xffLL & aKey[offset+15])<<56); //@fallthrough@ - case 15: b+=((0xffLL & aKey[offset+14])<<48); //@fallthrough@ - case 14: b+=((0xffLL & aKey[offset+13])<<40); //@fallthrough@ - case 13: b+=((0xffLL & aKey[offset+12])<<32); //@fallthrough@ - case 12: b+=((0xffLL & aKey[offset+11])<<24); //@fallthrough@ - case 11: b+=((0xffLL & aKey[offset+10])<<16); //@fallthrough@ - case 10: b+=((0xffLL & aKey[offset+ 9])<<8); //@fallthrough@ - case 9: b+=( 0xffLL & aKey[offset+ 8]); //@fallthrough@ - case 8: a+=((0xffLL & aKey[offset+ 7])<<56); //@fallthrough@ - case 7: a+=((0xffLL & aKey[offset+ 6])<<48); //@fallthrough@ - case 6: a+=((0xffLL & aKey[offset+ 5])<<40); //@fallthrough@ - case 5: a+=((0xffLL & aKey[offset+ 4])<<32); //@fallthrough@ - case 4: a+=((0xffLL & aKey[offset+ 3])<<24); //@fallthrough@ - case 3: a+=((0xffLL & aKey[offset+ 2])<<16); //@fallthrough@ - case 2: a+=((0xffLL & aKey[offset+ 1])<<8); //@fallthrough@ - case 1: a+=( 0xffLL & aKey[offset+ 0]); //@fallthrough@ + case 16: b+=((0xffLL & aKey[offset+15])<<56); [[fallthrough]]; + case 15: b+=((0xffLL & aKey[offset+14])<<48); [[fallthrough]]; + case 14: b+=((0xffLL & aKey[offset+13])<<40); [[fallthrough]]; + case 13: b+=((0xffLL & aKey[offset+12])<<32); [[fallthrough]]; + case 12: b+=((0xffLL & aKey[offset+11])<<24); [[fallthrough]]; + case 11: b+=((0xffLL & aKey[offset+10])<<16); [[fallthrough]]; + case 10: b+=((0xffLL & aKey[offset+ 9])<<8); [[fallthrough]]; + case 9: b+=( 0xffLL & aKey[offset+ 8]); [[fallthrough]]; + case 8: a+=((0xffLL & aKey[offset+ 7])<<56); [[fallthrough]]; + case 7: a+=((0xffLL & aKey[offset+ 6])<<48); [[fallthrough]]; + case 6: a+=((0xffLL & aKey[offset+ 5])<<40); [[fallthrough]]; + case 5: a+=((0xffLL & aKey[offset+ 4])<<32); [[fallthrough]]; + case 4: a+=((0xffLL & aKey[offset+ 3])<<24); [[fallthrough]]; + case 3: a+=((0xffLL & aKey[offset+ 2])<<16); [[fallthrough]]; + case 2: a+=((0xffLL & aKey[offset+ 1])<<8); [[fallthrough]]; + case 1: a+=( 0xffLL & aKey[offset+ 0]); // case 0: nothing left to add } diff --git a/searchlib/src/vespa/searchlib/query/query.cpp b/searchlib/src/vespa/searchlib/query/query.cpp index d380c273d02..984337f40ba 100644 --- a/searchlib/src/vespa/searchlib/query/query.cpp +++ b/searchlib/src/vespa/searchlib/query/query.cpp @@ -102,10 +102,11 @@ QueryConnector::create(ParseItem::ItemType type) case search::ParseItem::ITEM_WAND: return new OrQueryNode(); case search::ParseItem::ITEM_NOT: return new AndNotQueryNode(); case search::ParseItem::ITEM_PHRASE: return new PhraseQueryNode(); + case search::ParseItem::ITEM_SAME_ELEMENT: return new AndQueryNode(); // TODO: This needs a same element operation to work for streaming search too. case search::ParseItem::ITEM_NEAR: return new NearQueryNode(); case search::ParseItem::ITEM_ONEAR: return new ONearQueryNode(); default: - return NULL; + return nullptr; } } diff --git a/searchlib/src/vespa/searchlib/query/querynode.cpp b/searchlib/src/vespa/searchlib/query/querynode.cpp index 2e9d8be95f4..0d0a06de7af 100644 --- a/searchlib/src/vespa/searchlib/query/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/querynode.cpp @@ -26,6 +26,7 @@ QueryNode::UP QueryNode::Build(const QueryNode * parent, const QueryNodeResultFa case search::ParseItem::ITEM_WAND: case search::ParseItem::ITEM_NOT: case search::ParseItem::ITEM_PHRASE: + case search::ParseItem::ITEM_SAME_ELEMENT: case search::ParseItem::ITEM_NEAR: case search::ParseItem::ITEM_ONEAR: { diff --git a/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h b/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h index 72763839b95..611406e6ea3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/customtypetermvisitor.h @@ -5,8 +5,7 @@ #include "customtypevisitor.h" #include "intermediate.h" -namespace search { -namespace query { +namespace search::query { template <class NodeTypes> class CustomTypeTermVisitor : public CustomTypeVisitor<NodeTypes> @@ -27,10 +26,10 @@ private: void visit(typename NodeTypes::Or &n) override { visitChildren(n); } void visit(typename NodeTypes::Rank &n) override { visitChildren(n); } void visit(typename NodeTypes::WeakAnd &n) override { visitChildren(n); } + void visit(typename NodeTypes::SameElement &n) override { visitChildren(n); } // phrases and weighted set terms are conceptual leaf nodes and // should be handled that way. }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h index 83e6343e223..cdeebcaf9e5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h @@ -4,8 +4,7 @@ #include "queryvisitor.h" -namespace search { -namespace query { +namespace search::query { /** * By typedefing a (complete) set of subclasses to the query nodes in @@ -37,6 +36,7 @@ public: virtual void visit(typename NodeTypes::ONear &) = 0; virtual void visit(typename NodeTypes::Or &) = 0; virtual void visit(typename NodeTypes::Phrase &) = 0; + virtual void visit(typename NodeTypes::SameElement &) = 0; virtual void visit(typename NodeTypes::PrefixTerm &) = 0; virtual void visit(typename NodeTypes::RangeTerm &) = 0; virtual void visit(typename NodeTypes::Rank &) = 0; @@ -62,6 +62,7 @@ private: typedef typename NodeTypes::ONear TONear; typedef typename NodeTypes::Or TOr; typedef typename NodeTypes::Phrase TPhrase; + typedef typename NodeTypes::SameElement TSameElement; typedef typename NodeTypes::PrefixTerm TPrefixTerm; typedef typename NodeTypes::RangeTerm TRangeTerm; typedef typename NodeTypes::Rank TRank; @@ -84,6 +85,7 @@ private: void visit(ONear &n) override { visit(static_cast<TONear&>(n)); } void visit(Or &n) override { visit(static_cast<TOr&>(n)); } void visit(Phrase &n) override { visit(static_cast<TPhrase&>(n)); } + void visit(SameElement &n) override { visit(static_cast<TSameElement &>(n)); } void visit(PrefixTerm &n) override { visit(static_cast<TPrefixTerm&>(n)); } void visit(RangeTerm &n) override { visit(static_cast<TRangeTerm&>(n)); } void visit(Rank &n) override { visit(static_cast<TRank&>(n)); } @@ -98,5 +100,4 @@ private: void visit(RegExpTerm &n) override { visit(static_cast<TRegExpTerm&>(n)); } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp b/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp index 1274777dc74..f56da9c2cf9 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/intermediate.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "intermediate.h" -namespace search { -namespace query { +namespace search::query { Intermediate::~Intermediate() { for (size_t i = 0; i < _children.size(); ++i) { @@ -16,5 +15,4 @@ Intermediate &Intermediate::append(Node::UP child) return *this; } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediate.h b/searchlib/src/vespa/searchlib/query/tree/intermediate.h index c28ab8241e0..052dc1db269 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediate.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediate.h @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "node.h" #include <vector> -#include <vespa/searchlib/query/tree/node.h> -namespace search { -namespace query { +namespace search::query { class Intermediate : public Node { @@ -24,6 +23,4 @@ class Intermediate : public Node Intermediate &append(Node::UP child); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp index 9fabbe58a19..4a4b606ef8f 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp @@ -1,32 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "intermediatenodes.h" -namespace search { -namespace query { - -And::~And() {} - -AndNot::~AndNot() {} - -Or::~Or() {} - -WeakAnd::~WeakAnd() {} - -Equiv::~Equiv() {} - -Rank::~Rank() {} - -Near::~Near() {} - -ONear::~ONear() {} - -Phrase::~Phrase() {} - -WeightedSetTerm::~WeightedSetTerm() {} - -DotProduct::~DotProduct() {} - -WandTerm::~WandTerm() {} - -} // namespace query -} // namespace search +namespace search::query { + +And::~And() = default; +AndNot::~AndNot() = default; +Or::~Or() = default; +WeakAnd::~WeakAnd() = default; +Equiv::~Equiv() = default; +Rank::~Rank() = default; +Near::~Near() = default; +ONear::~ONear() = default; +Phrase::~Phrase() = default; +SameElement::~SameElement() = default; +WeightedSetTerm::~WeightedSetTerm() = default; +DotProduct::~DotProduct() = default; +WandTerm::~WandTerm() = default; + +} diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h index e358c7d7bef..6d643d951f0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h @@ -5,10 +5,8 @@ #include "intermediate.h" #include "querynodemixin.h" #include "term.h" -#include <vespa/searchlib/query/weight.h> -namespace search { -namespace query { +namespace search::query { class And : public QueryNodeMixin<And, Intermediate> { public: @@ -105,6 +103,15 @@ public: virtual ~Phrase() = 0; }; +class SameElement : public QueryNodeMixin<SameElement, Intermediate> { +public: + SameElement(const vespalib::string &view) : _view(view) {} + virtual ~SameElement() = 0; + const vespalib::string & getView() const { return _view; } +private: + vespalib::string _view; +}; + class WeightedSetTerm : public QueryNodeMixin<WeightedSetTerm, Intermediate>, public Term { public: WeightedSetTerm(const vespalib::string &view, int32_t id, Weight weight) @@ -137,6 +144,4 @@ public: double getThresholdBoostFactor() const { return _thresholdBoostFactor; } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/location.cpp b/searchlib/src/vespa/searchlib/query/tree/location.cpp index 5817b93a2fb..216c5ec5ad0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/location.cpp @@ -7,8 +7,7 @@ using vespalib::asciistream; -namespace search { -namespace query { +namespace search::query { Location::Location(const Point &point, uint32_t max_dist, uint32_t x_aspect) { asciistream loc; @@ -61,5 +60,4 @@ vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &lo return out << loc.getLocationString(); } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/location.h b/searchlib/src/vespa/searchlib/query/tree/location.h index 0abad19b696..5ed717c87d6 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.h +++ b/searchlib/src/vespa/searchlib/query/tree/location.h @@ -4,11 +4,8 @@ #include <vespa/vespalib/stllike/string.h> -namespace vespalib { - class asciistream; -} -namespace search { -namespace query { +namespace vespalib { class asciistream; } +namespace search::query { class Point; class Rectangle; @@ -32,6 +29,4 @@ public: vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc); -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/point.h b/searchlib/src/vespa/searchlib/query/tree/point.h index af9b958a474..89d0bc1db44 100644 --- a/searchlib/src/vespa/searchlib/query/tree/point.h +++ b/searchlib/src/vespa/searchlib/query/tree/point.h @@ -2,10 +2,9 @@ #pragma once -#include <stdint.h> +#include <cstdint> -namespace search { -namespace query { +namespace search::query { struct Point { int64_t x; @@ -18,6 +17,4 @@ inline bool operator==(const Point &p1, const Point &p2) { return p1.x == p2.x && p1.y == p2.y; } -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h index b851563e198..0a92546e414 100644 --- a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h +++ b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h @@ -6,8 +6,7 @@ #include <memory> #include <vector> -namespace search { -namespace query { +namespace search::query { /** * Represents a predicate query, with features and range features. @@ -71,6 +70,4 @@ public: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 2732d7895a5..3c6ff93457d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -20,13 +20,11 @@ #pragma once #include "predicate_query_term.h" -#include <stack> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/query/weight.h> #include "node.h" +#include <vespa/searchlib/query/weight.h> +#include <stack> -namespace search { -namespace query { +namespace search::query { class Intermediate; class Location; @@ -124,6 +122,10 @@ typename NodeTypes::Phrase *createPhrase(const vespalib::stringref &view, int32_ return new typename NodeTypes::Phrase(view, id, weight); } template <class NodeTypes> +typename NodeTypes::SameElement *createSameElement(const vespalib::stringref &view) { + return new typename NodeTypes::SameElement(view); +} +template <class NodeTypes> typename NodeTypes::WeightedSetTerm *createWeightedSetTerm(const vespalib::stringref &view, int32_t id, Weight weight) { return new typename NodeTypes::WeightedSetTerm(view, id, weight); } @@ -243,6 +245,9 @@ public: setWeightOverride(weight); return node; } + typename NodeTypes::SameElement &addSameElement(int child_count, const stringref &view) { + return addIntermediate(createSameElement<NodeTypes>(view), child_count); + } typename NodeTypes::WeightedSetTerm &addWeightedSetTerm( int child_count, const stringref &view, int32_t id, Weight weight) { adjustWeight(weight); typename NodeTypes::WeightedSetTerm &node = addIntermediate(createWeightedSetTerm<NodeTypes>(view, id, weight), child_count); @@ -306,6 +311,4 @@ public: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h index d6e0950dea2..9e8c97cff94 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h +++ b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h @@ -4,8 +4,7 @@ #include "queryvisitor.h" -namespace search { -namespace query { +namespace search::query { template <typename T, typename Base> struct QueryNodeMixin : Base { @@ -21,8 +20,6 @@ protected: }; template <typename T, typename Base> -QueryNodeMixin<T, Base>::~QueryNodeMixin() {} - -} // namespace query -} // namespace search +QueryNodeMixin<T, Base>::~QueryNodeMixin() = default; +} diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index aad85d9b28c..7bf6c17f136 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -7,8 +7,7 @@ #include "queryvisitor.h" #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { /** * Creates a new query tree based on an existing one. The traits class @@ -70,22 +69,24 @@ private: } void visit(Phrase &node) override { - replicate(node, _builder.addPhrase(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addPhrase(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } + void visit(SameElement &node) override { + _builder.addSameElement(node.getChildren().size(), node.getView()); + visitNodes(node.getChildren()); + } + void visit(WeightedSetTerm &node) override { - replicate(node, _builder.addWeightedSetTerm(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addWeightedSetTerm(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } void visit(DotProduct &node) override { - replicate(node, _builder.addDotProduct(node.getChildren().size(), - node.getView(), + replicate(node, _builder.addDotProduct(node.getChildren().size(), node.getView(), node.getId(), node.getWeight())); visitNodes(node.getChildren()); } @@ -165,5 +166,4 @@ private: } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h b/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h index a3e6ac523ae..c42a16d8ab3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/querytreecreator.h @@ -5,8 +5,7 @@ #include "queryreplicator.h" #include "stackdumpquerycreator.h" -namespace search { -namespace query { +namespace search::query { /** * Holds functions for creating query trees, either from a stack dump @@ -27,6 +26,4 @@ private: QueryTreeCreator(); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h index 4596359201b..0cb56f9127a 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace query { +namespace search::query { class And; class AndNot; @@ -26,6 +25,7 @@ class DotProduct; class WandTerm; class PredicateQuery; class RegExpTerm; +class SameElement; struct QueryVisitor { virtual ~QueryVisitor() {} @@ -39,6 +39,7 @@ struct QueryVisitor { virtual void visit(ONear &) = 0; virtual void visit(Or &) = 0; virtual void visit(Phrase &) = 0; + virtual void visit(SameElement &node) = 0; virtual void visit(PrefixTerm &) = 0; virtual void visit(RangeTerm &) = 0; virtual void visit(Rank &) = 0; @@ -53,6 +54,5 @@ struct QueryVisitor { virtual void visit(RegExpTerm &) = 0; }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/range.cpp b/searchlib/src/vespa/searchlib/query/tree/range.cpp index 583d57bb74d..202b80d98a2 100644 --- a/searchlib/src/vespa/searchlib/query/tree/range.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/range.cpp @@ -3,8 +3,7 @@ #include "range.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace query { +namespace search::query { Range::Range(int64_t f, int64_t t) { @@ -18,5 +17,4 @@ vespalib::asciistream &operator<<(vespalib::asciistream &out, const Range &range return out << range.getRangeString(); } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/range.h b/searchlib/src/vespa/searchlib/query/tree/range.h index 862258873e6..e55ddf7f14b 100644 --- a/searchlib/src/vespa/searchlib/query/tree/range.h +++ b/searchlib/src/vespa/searchlib/query/tree/range.h @@ -3,12 +3,9 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } -namespace search { -namespace query { +namespace search::query { class Range { vespalib::string _range; @@ -27,6 +24,4 @@ inline bool operator==(const Range &r1, const Range &r2) { vespalib::asciistream &operator<<(vespalib::asciistream &out, const Range &range); -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/rectangle.h b/searchlib/src/vespa/searchlib/query/tree/rectangle.h index 29b144ac5dc..97be9ddeb32 100644 --- a/searchlib/src/vespa/searchlib/query/tree/rectangle.h +++ b/searchlib/src/vespa/searchlib/query/tree/rectangle.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace query { +namespace search::query { struct Rectangle { int64_t left; @@ -21,6 +20,5 @@ inline bool operator==(const Rectangle &r1, const Rectangle &r2) { && r1.top == r2.top && r1.bottom == r2.bottom; } -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index 28791dafe53..557d0964bcb 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -10,8 +10,7 @@ #include "intermediatenodes.h" #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { struct SimpleAnd : And {}; struct SimpleAndNot : AndNot {}; @@ -31,6 +30,10 @@ struct SimplePhrase : Phrase { SimplePhrase(const vespalib::stringref &view, int32_t id, Weight weight) : Phrase(view, id, weight) {} }; + +struct SimpleSameElement : SameElement { + SimpleSameElement(const vespalib::stringref &view) : SameElement(view) {} +}; struct SimpleWeightedSetTerm : WeightedSetTerm { SimpleWeightedSetTerm(const vespalib::stringref &view, int32_t id, Weight weight) : WeightedSetTerm(view, id, weight) {} @@ -112,6 +115,7 @@ struct SimpleQueryNodeTypes { typedef SimpleONear ONear; typedef SimpleOr Or; typedef SimplePhrase Phrase; + typedef SimpleSameElement SameElement; typedef SimplePrefixTerm PrefixTerm; typedef SimpleRangeTerm RangeTerm; typedef SimpleRank Rank; @@ -126,6 +130,4 @@ struct SimpleQueryNodeTypes { typedef SimpleRegExpTerm RegExpTerm; }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 771830c0b03..645750b8576 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -64,23 +64,48 @@ class QueryNodeConverter : public QueryVisitor { template <typename V> void appendPredicateQueryTermVector(const V& v); + void createComplexIntermediate(const Term &node, const std::vector<Node *> & children, size_t type) { + uint8_t flags = 0; + if (!node.isRanked()) { + flags |= ParseItem::IFLAG_NORANK; + } + if (!node.usePositionData()) { + flags |= ParseItem::IFLAG_NOPOSITIONDATA; + } + if (flags != 0) { + type |= ParseItem::IF_FLAGS; + } + appendByte(type); + appendCompressedNumber(node.getWeight().percent()); + if (type & ParseItem::IF_FLAGS) { + appendByte(flags); + } + appendCompressedPositiveNumber(children.size()); + appendString(node.getView()); + visitNodes(children); + } + void createIntermediate(const Intermediate &node, size_t type) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, - size_t distance) { + void createIntermediate(const Intermediate &node, size_t type, size_t distance) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); appendCompressedPositiveNumber(distance); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, - size_t distance, - const vespalib::string & view) { + void createIntermediate(const Intermediate &node, size_t type, const vespalib::string & view) { + appendByte(type); + appendCompressedPositiveNumber(node.getChildren().size()); + appendString(view); + visitNodes(node.getChildren()); + } + + void createIntermediate(const Intermediate &node, size_t type, size_t distance, const vespalib::string & view) { appendByte(type); appendCompressedPositiveNumber(node.getChildren().size()); appendCompressedPositiveNumber(distance); @@ -116,26 +141,12 @@ class QueryNodeConverter : public QueryVisitor { createIntermediate(node, ParseItem::ITEM_EQUIV); } + void visit(SameElement &node) override { + createIntermediate(node, ParseItem::ITEM_SAME_ELEMENT, node.getView()); + } + void visit(Phrase &node) override { - uint8_t typefield = (ParseItem::ITEM_PHRASE | ParseItem::IF_WEIGHT); - uint8_t flags = 0; - if (!node.isRanked()) { - flags |= ParseItem::IFLAG_NORANK; - } - if (!node.usePositionData()) { - flags |= ParseItem::IFLAG_NOPOSITIONDATA; - } - if (flags != 0) { - typefield |= ParseItem::IF_FLAGS; - } - appendByte(typefield); - appendCompressedNumber(node.getWeight().percent()); - if (typefield & ParseItem::IF_FLAGS) { - appendByte(flags); - } - appendCompressedPositiveNumber(node.getChildren().size()); - appendString(node.getView()); - visitNodes(node.getChildren()); + createComplexIntermediate(node, node.getChildren(), (ParseItem::ITEM_PHRASE | ParseItem::IF_WEIGHT)); } template <typename NODE> @@ -187,9 +198,7 @@ class QueryNodeConverter : public QueryVisitor { template <class Term> void createTerm(const Term &node, size_t type) { - uint8_t typefield = type | - ParseItem::IF_WEIGHT | - ParseItem::IF_UNIQUEID; + uint8_t typefield = type | ParseItem::IF_WEIGHT | ParseItem::IF_UNIQUEID; uint8_t flags = 0; if (!node.isRanked()) { flags |= ParseItem::IFLAG_NORANK; diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h index 448f1c9bd08..4e1556d05e6 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.h @@ -4,8 +4,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace query { +namespace search::query { class Node; @@ -14,6 +13,4 @@ struct StackDumpCreator { static vespalib::string create(const Node &node); }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index f9c66965cc4..fa42cdac1c0 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -9,8 +9,7 @@ #include <vespa/searchlib/parsequery/simplequerystack.h> #include <vespa/vespalib/objects/hexdump.h> -namespace search { -namespace query { +namespace search::query { /** * Creates a query tree from a stack dump. @@ -90,6 +89,10 @@ private: Weight weight = queryStack.GetWeight(); t = &builder.addPhrase(arity, view, id, weight); pureTermView = view; + } else if (type == ParseItem::ITEM_SAME_ELEMENT) { + vespalib::stringref view = queryStack.getIndexName(); + builder.addSameElement(arity, view); + pureTermView = view; } else if (type == ParseItem::ITEM_WEIGHTED_SET) { vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); @@ -152,6 +155,4 @@ private: } }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h index e4ce4ccf807..0cdaca82572 100644 --- a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h @@ -4,8 +4,7 @@ #include "customtypetermvisitor.h" -namespace search { -namespace query { +namespace search::query { /** * Use this class to visit all term nodes by deriving from this class @@ -54,5 +53,4 @@ class TemplateTermVisitor : public CustomTypeTermVisitor<NodeTypes> { void visit(typename NodeTypes::WandTerm &n) override { myVisit(n); } }; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/term.cpp b/searchlib/src/vespa/searchlib/query/tree/term.cpp index 54def08afe8..de59752aa10 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/term.cpp @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "term.h" +#include <cassert> -namespace search { -namespace query { +namespace search::query { -Term::~Term() { } +Term::~Term() = default; Term::Term(const vespalib::stringref &view, int32_t id, Weight weight) : _view(view), @@ -16,5 +16,14 @@ Term::Term(const vespalib::stringref &view, int32_t id, Weight weight) : _position_data(true) { } -} // namespace query -} // namespace search +void Term::setStateFrom(const Term& other) { + setTermIndex(other.getTermIndex()); + setRanked(other.isRanked()); + setPositionData(other.usePositionData()); + // too late to copy this state: + assert(_view == other.getView()); + assert(_id == other.getId()); + assert(_weight == other.getWeight()); +} + +} diff --git a/searchlib/src/vespa/searchlib/query/tree/term.h b/searchlib/src/vespa/searchlib/query/tree/term.h index f8c0d98ac22..8e9b9897e9d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.h +++ b/searchlib/src/vespa/searchlib/query/tree/term.h @@ -1,13 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/query/tree/node.h> +#include "node.h" #include <vespa/searchlib/query/weight.h> -#include <cassert> +#include <vespa/vespalib/stllike/string.h> -namespace search { -namespace query { +namespace search::query { /** * This is a leaf in the Query tree. Sort of. Phrases are both terms @@ -16,11 +14,11 @@ namespace query { class Term { vespalib::string _view; - int32_t _id; - Weight _weight; - int32_t _term_index; - bool _ranked; - bool _position_data; + int32_t _id; + Weight _weight; + int32_t _term_index; + bool _ranked; + bool _position_data; public: virtual ~Term() = 0; @@ -29,15 +27,7 @@ public: void setRanked(bool ranked) { _ranked = ranked; } void setPositionData(bool position_data) { _position_data = position_data; } - void setStateFrom(const Term& other) { - setTermIndex(other.getTermIndex()); - setRanked(other.isRanked()); - setPositionData(other.usePositionData()); - // too late to copy this state: - assert(_view == other.getView()); - assert(_id == other.getId()); - assert(_weight == other.getWeight()); - } + void setStateFrom(const Term& other); const vespalib::string & getView() const { return _view; } Weight getWeight() const { return _weight; } @@ -60,7 +50,7 @@ class TermBase : public Node, public Term { public: typedef T Type; - virtual ~TermBase() = 0; + ~TermBase() override = 0; const T &getTerm() const { return _term; } protected: @@ -71,8 +61,6 @@ protected: }; template <typename T> -TermBase<T>::~TermBase() {} - -} // namespace query -} // namespace search +TermBase<T>::~TermBase() = default; +} diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index 8e8ae16827b..0a6a6af62b5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -2,27 +2,25 @@ #include "termnodes.h" -namespace search { -namespace query { +namespace search::query { -NumberTerm::~NumberTerm() {} +NumberTerm::~NumberTerm() = default; -PrefixTerm::~PrefixTerm() {} +PrefixTerm::~PrefixTerm() = default; -RangeTerm::~RangeTerm() {} +RangeTerm::~RangeTerm() = default; StringTerm::StringTerm(const Type &term, const vespalib::stringref &view, int32_t id, Weight weight) : QueryNodeMixinType(term, view, id, weight) {} -StringTerm::~StringTerm() {} +StringTerm::~StringTerm() = default; -SubstringTerm::~SubstringTerm() {} +SubstringTerm::~SubstringTerm() = default; -SuffixTerm::~SuffixTerm() {} +SuffixTerm::~SuffixTerm() = default; -LocationTerm::~LocationTerm() {} +LocationTerm::~LocationTerm() = default; -RegExpTerm::~RegExpTerm() {} +RegExpTerm::~RegExpTerm() = default; -} // namespace query -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index 4c98ba92ff5..8d4882fb393 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -8,8 +8,7 @@ #include "range.h" #include "term.h" -namespace search { -namespace query { +namespace search::query { typedef TermBase<vespalib::string> StringBase; @@ -117,6 +116,4 @@ public: }; -} // namespace query -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/query/weight.h b/searchlib/src/vespa/searchlib/query/weight.h index 1b4168231c5..51596642cd2 100644 --- a/searchlib/src/vespa/searchlib/query/weight.h +++ b/searchlib/src/vespa/searchlib/query/weight.h @@ -3,8 +3,7 @@ #include <cstdint> -namespace search { -namespace query { +namespace search::query { /** * Represents the weight given on a query item such as a term, phrase, or equiv. @@ -44,8 +43,7 @@ public: bool operator== (const Weight& other) const { return _weight == other._weight; } }; -} // namespace query -} // namespace search +} inline search::query::Weight operator+(const search::query::Weight& a, const search::query::Weight& b) { diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 683de780107..ecda6d6d6ef 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -33,6 +33,8 @@ vespa_add_library(searchlib_queryeval OBJECT predicate_blueprint.cpp predicate_search.cpp ranksearch.cpp + same_element_blueprint.cpp + same_element_search.cpp searchable.cpp searchiterator.cpp simple_phrase_blueprint.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h index ea051aa9318..7cf323aa106 100644 --- a/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h +++ b/searchlib/src/vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h @@ -5,8 +5,7 @@ #include "searchiterator.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -namespace search { -namespace queryeval { +namespace search::queryeval { /** * A term iterator wrapper used to hide detailed match @@ -22,16 +21,14 @@ private: SearchIterator::UP _search; fef::TermFieldMatchData *_tfmdp; - BooleanMatchIteratorWrapper(const BooleanMatchIteratorWrapper &); - BooleanMatchIteratorWrapper &operator=(const BooleanMatchIteratorWrapper &); - protected: void doSeek(uint32_t docid) override; void doUnpack(uint32_t docid) override; Trinary is_strict() const override { return _search->is_strict(); } void initRange(uint32_t beginid, uint32_t endid) override { + SearchIterator::initRange(beginid, endid); _search->initRange(beginid, endid); - SearchIterator::initRange(_search->getDocId()+1, _search->getEndId()); + setDocId(_search->getDocId()); } public: @@ -49,12 +46,9 @@ public: * @param search internal search, must be a term iterator * @param match term match data used by the internal iterator **/ - BooleanMatchIteratorWrapper(SearchIterator::UP search, - const fef::TermFieldMatchDataArray &matchData); + BooleanMatchIteratorWrapper(SearchIterator::UP search, const fef::TermFieldMatchDataArray &matchData); void visitMembers(vespalib::ObjectVisitor &visitor) const override; }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index 8e6429aaa90..e3dac98e588 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -19,7 +19,7 @@ CreateBlueprintVisitorHelper::CreateBlueprintVisitorHelper(Searchable &searchabl _result() {} -CreateBlueprintVisitorHelper::~CreateBlueprintVisitorHelper() {} +CreateBlueprintVisitorHelper::~CreateBlueprintVisitorHelper() = default; Blueprint::UP CreateBlueprintVisitorHelper::getResult() @@ -30,7 +30,7 @@ CreateBlueprintVisitorHelper::getResult() } void -CreateBlueprintVisitorHelper::visitPhrase(search::query::Phrase &n) { +CreateBlueprintVisitorHelper::visitPhrase(query::Phrase &n) { SimplePhraseBlueprint *phrase = new SimplePhraseBlueprint(_field, _requestContext); Blueprint::UP result(phrase); for (size_t i = 0; i < n.getChildren().size(); ++i) { @@ -42,7 +42,7 @@ CreateBlueprintVisitorHelper::visitPhrase(search::query::Phrase &n) { } void -CreateBlueprintVisitorHelper::handleNumberTermAsText(search::query::NumberTerm &n) +CreateBlueprintVisitorHelper::handleNumberTermAsText(query::NumberTerm &n) { vespalib::string termStr = termAsString(n); queryeval::SplitFloat splitter(termStr); @@ -73,24 +73,24 @@ CreateBlueprintVisitorHelper::createWeightedSet(WS *bp, NODE &n) { for (size_t i = 0; i < n.getChildren().size(); ++i) { fields.clear(); fields.add(bp->getNextChildField(_field)); - const search::query::Node &node = *n.getChildren()[i]; + const query::Node &node = *n.getChildren()[i]; uint32_t weight = getWeightFromNode(node).percent(); bp->addTerm(_searchable.createBlueprint(_requestContext, fields, node), weight); } setResult(std::move(result)); } void -CreateBlueprintVisitorHelper::visitWeightedSetTerm(search::query::WeightedSetTerm &n) { +CreateBlueprintVisitorHelper::visitWeightedSetTerm(query::WeightedSetTerm &n) { WeightedSetTermBlueprint *bp = new WeightedSetTermBlueprint(_field); createWeightedSet(bp, n); } void -CreateBlueprintVisitorHelper::visitDotProduct(search::query::DotProduct &n) { +CreateBlueprintVisitorHelper::visitDotProduct(query::DotProduct &n) { DotProductBlueprint *bp = new DotProductBlueprint(_field); createWeightedSet(bp, n); } void -CreateBlueprintVisitorHelper::visitWandTerm(search::query::WandTerm &n) { +CreateBlueprintVisitorHelper::visitWandTerm(query::WandTerm &n) { ParallelWeakAndBlueprint *bp = new ParallelWeakAndBlueprint(_field, n.getTargetNumHits(), n.getScoreThreshold(), diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h index cded9c103dc..5bcc4f4c4c5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h @@ -12,7 +12,7 @@ namespace search::queryeval { -class CreateBlueprintVisitorHelper : public search::query::QueryVisitor +class CreateBlueprintVisitorHelper : public query::QueryVisitor { private: const IRequestContext & _requestContext; @@ -37,42 +37,43 @@ public: const FieldSpec &getField() const { return _field; } - void visitPhrase(search::query::Phrase &n); + void visitPhrase(query::Phrase &n); template <typename WS, typename NODE> void createWeightedSet(WS *bp, NODE &n); - void visitWeightedSetTerm(search::query::WeightedSetTerm &n); - void visitDotProduct(search::query::DotProduct &n); - void visitWandTerm(search::query::WandTerm &n); + void visitWeightedSetTerm(query::WeightedSetTerm &n); + void visitDotProduct(query::DotProduct &n); + void visitWandTerm(query::WandTerm &n); - void handleNumberTermAsText(search::query::NumberTerm &n); + void handleNumberTermAsText(query::NumberTerm &n); void illegalVisit() {} - void visit(search::query::And &) override { illegalVisit(); } - void visit(search::query::AndNot &) override { illegalVisit(); } - void visit(search::query::Equiv &) override { illegalVisit(); } - void visit(search::query::Near &) override { illegalVisit(); } - void visit(search::query::ONear &) override { illegalVisit(); } - void visit(search::query::Or &) override { illegalVisit(); } - void visit(search::query::Rank &) override { illegalVisit(); } - void visit(search::query::WeakAnd &) override { illegalVisit(); } - - void visit(search::query::Phrase &n) override { + void visit(query::And &) override { illegalVisit(); } + void visit(query::AndNot &) override { illegalVisit(); } + void visit(query::Equiv &) override { illegalVisit(); } + void visit(query::Near &) override { illegalVisit(); } + void visit(query::ONear &) override { illegalVisit(); } + void visit(query::Or &) override { illegalVisit(); } + void visit(query::Rank &) override { illegalVisit(); } + void visit(query::WeakAnd &) override { illegalVisit(); } + void visit(query::SameElement &) override { illegalVisit(); } + + void visit(query::Phrase &n) override { visitPhrase(n); } - void visit(search::query::WeightedSetTerm &n) override { visitWeightedSetTerm(n); } - void visit(search::query::DotProduct &n) override { visitDotProduct(n); } - void visit(search::query::WandTerm &n) override { visitWandTerm(n); } - - void visit(search::query::NumberTerm &n) override = 0; - void visit(search::query::LocationTerm &n) override = 0; - void visit(search::query::PrefixTerm &n) override = 0; - void visit(search::query::RangeTerm &n) override = 0; - void visit(search::query::StringTerm &n) override = 0; - void visit(search::query::SubstringTerm &n) override = 0; - void visit(search::query::SuffixTerm &n) override = 0; - void visit(search::query::RegExpTerm &n) override = 0; + void visit(query::WeightedSetTerm &n) override { visitWeightedSetTerm(n); } + void visit(query::DotProduct &n) override { visitDotProduct(n); } + void visit(query::WandTerm &n) override { visitWandTerm(n); } + + void visit(query::NumberTerm &n) override = 0; + void visit(query::LocationTerm &n) override = 0; + void visit(query::PrefixTerm &n) override = 0; + void visit(query::RangeTerm &n) override = 0; + void visit(query::StringTerm &n) override = 0; + void visit(query::SubstringTerm &n) override = 0; + void visit(query::SuffixTerm &n) override = 0; + void visit(query::RegExpTerm &n) override = 0; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp index 8fa6af74ae2..bd9de0a1762 100644 --- a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp @@ -13,8 +13,7 @@ using search::query::Weight; namespace search::queryeval { namespace { -struct WeightExtractor : public TemplateTermVisitor<WeightExtractor, - SimpleQueryNodeTypes> { +struct WeightExtractor : public TemplateTermVisitor<WeightExtractor, SimpleQueryNodeTypes> { Weight weight; WeightExtractor() : weight(0) {} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp new file mode 100644 index 00000000000..840bb1e5ea9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -0,0 +1,98 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_blueprint.h" +#include "same_element_search.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/attribute/elementiterator.h> +#include <vespa/vespalib/objects/visit.hpp> +#include <algorithm> +#include <map> + +namespace search::queryeval { + +SameElementBlueprint::SameElementBlueprint() + : ComplexLeafBlueprint(FieldSpecBaseList()), + _estimate(), + _layout(), + _terms() +{ +} + +FieldSpec +SameElementBlueprint::getNextChildField(const vespalib::string &field_name, uint32_t field_id) +{ + return FieldSpec(field_name, field_id, _layout.allocTermField(field_id), false); +} + +void +SameElementBlueprint::addTerm(Blueprint::UP term) +{ + const State &childState = term->getState(); + assert(childState.numFields() == 1); + HitEstimate childEst = childState.estimate(); + if (_terms.empty() || (childEst < _estimate)) { + _estimate = childEst; + setEstimate(_estimate); + } + _terms.push_back(std::move(term)); +} + +void +SameElementBlueprint::optimize_self() +{ + std::sort(_terms.begin(), _terms.end(), + [](const auto &a, const auto &b) + { + return (a->getState().estimate() < b->getState().estimate()); + }); +} + +void +SameElementBlueprint::fetchPostings(bool strict) +{ + for (size_t i = 0; i < _terms.size(); ++i) { + _terms[i]->fetchPostings(strict && (i == 0)); + } +} + +SearchIterator::UP +SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, + bool strict) const +{ + (void) tfmda; + assert(!tfmda.valid()); + + fef::MatchDataLayout my_layout = _layout; + std::vector<fef::TermFieldHandle> extra_handles; + for (size_t i = 0; i < _terms.size(); ++i) { + const State &childState = _terms[i]->getState(); + assert(childState.numFields() == 1); + extra_handles.push_back(my_layout.allocTermField(childState.field(0).getFieldId())); + } + fef::MatchData::UP md = my_layout.createMatchData(); + search::fef::TermFieldMatchDataArray childMatch; + std::vector<SearchIterator::UP> children(_terms.size()); + for (size_t i = 0; i < _terms.size(); ++i) { + const State &childState = _terms[i]->getState(); + SearchIterator::UP child = _terms[i]->createSearch(*md, (strict && (i == 0))); + const attribute::ISearchContext *context = child->getAttributeSearchContext(); + if (context == nullptr) { + children[i] = std::move(child); + childMatch.add(childState.field(0).resolve(*md)); + } else { + fef::TermFieldMatchData *child_tfmd = md->resolveTermField(extra_handles[i]); + children[i] = std::make_unique<attribute::ElementIterator>(std::move(child), *context, *child_tfmd); + childMatch.add(child_tfmd); + } + } + return std::make_unique<SameElementSearch>(std::move(md), std::move(children), childMatch, strict); +} + +void +SameElementBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const +{ + ComplexLeafBlueprint::visitMembers(visitor); + visit(visitor, "terms", _terms); +} + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h new file mode 100644 index 00000000000..050a2bc31d4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h @@ -0,0 +1,43 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchable.h" +#include <vespa/searchlib/fef/matchdatalayout.h> + +namespace search::fef { class TermFieldMatchData; } + +namespace search::queryeval { + +class SameElementBlueprint : public ComplexLeafBlueprint +{ +private: + HitEstimate _estimate; + fef::MatchDataLayout _layout; + std::vector<Blueprint::UP> _terms; + +public: + SameElementBlueprint(); + SameElementBlueprint(const SameElementBlueprint &) = delete; + SameElementBlueprint &operator=(const SameElementBlueprint &) = delete; + ~SameElementBlueprint() = default; + + // no match data + bool isWhiteList() const override { return true; } + + // used by create visitor + FieldSpec getNextChildField(const vespalib::string &field_name, uint32_t field_id); + + // used by create visitor + void addTerm(Blueprint::UP term); + + void optimize_self() override; + void fetchPostings(bool strict) override; + + SearchIteratorUP createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, + bool strict) const override; + void visitMembers(vespalib::ObjectVisitor &visitor) const override; + const std::vector<Blueprint::UP> &terms() const { return _terms; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp new file mode 100644 index 00000000000..8f3fc9c350d --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -0,0 +1,117 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_search.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/objects/visit.h> +#include <vespa/vespalib/objects/visit.hpp> +#include <algorithm> +#include <functional> + +using TFMD = search::fef::TermFieldMatchData; + +namespace search::queryeval { + +namespace { + +template <typename It> +int32_t try_match(const fef::TermFieldMatchDataArray &match, std::vector<It> &iterators, uint32_t cand) { + for (size_t i = 0; i < iterators.size(); ++i) { + while ((iterators[i] != match[i]->end()) && (iterators[i]->getElementId() < cand)) { + ++iterators[i]; + } + if (iterators[i] == match[i]->end()) { + return -1; + } + if (iterators[i]->getElementId() != cand) { + return iterators[i]->getElementId(); + } + } + return cand; +} + +} + +bool +SameElementSearch::check_docid_match(uint32_t docid) +{ + for (const auto &child: _children) { + if (!child->seek(docid)) { + return false; + } + } + return true; +} + +void +SameElementSearch::unpack_children(uint32_t docid) +{ + for (const auto &child: _children) { + child->doUnpack(docid); + } + for (size_t i = 0; i < _childMatch.size(); ++i) { + _iterators[i] = _childMatch[i]->begin(); + } +} + +bool +SameElementSearch::check_element_match(uint32_t docid) +{ + unpack_children(docid); + int32_t cand = 0; + int32_t next = try_match(_childMatch, _iterators, cand); + while (next > cand) { + cand = next; + next = try_match(_childMatch, _iterators, cand); + } + return (cand == next); +} + +SameElementSearch::SameElementSearch(fef::MatchData::UP md, + std::vector<SearchIterator::UP> children, + const fef::TermFieldMatchDataArray &childMatch, + bool strict) + : _md(std::move(md)), + _children(std::move(children)), + _childMatch(childMatch), + _iterators(childMatch.size()), + _strict(strict) +{ + assert(!_children.empty()); + assert(_childMatch.valid()); +} + +void +SameElementSearch::initRange(uint32_t begin_id, uint32_t end_id) +{ + SearchIterator::initRange(begin_id, end_id); + for (const auto &child: _children) { + child->initRange(begin_id, end_id); + } +} + +void +SameElementSearch::doSeek(uint32_t docid) { + if (check_docid_match(docid) && check_element_match(docid)) { + setDocId(docid); + } else if (_strict) { + docid = std::max(docid + 1, _children[0]->getDocId()); + while (!isAtEnd(docid)) { + if (check_docid_match(docid) && check_element_match(docid)) { + setDocId(docid); + return; + } + docid = std::max(docid + 1, _children[0]->getDocId()); + } + setAtEnd(); + } +} + +void +SameElementSearch::visitMembers(vespalib::ObjectVisitor &visitor) const +{ + SearchIterator::visitMembers(visitor); + visit(visitor, "children", _children); + visit(visitor, "strict", _strict); +} + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h new file mode 100644 index 00000000000..6a116c76e73 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h @@ -0,0 +1,44 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <memory> +#include <vector> + +namespace search::queryeval { + +/** + * Search iterator for a collection of terms that need to match within + * the same element (array index). + */ +class SameElementSearch : public SearchIterator +{ +private: + using It = fef::TermFieldMatchData::PositionsIterator; + + fef::MatchData::UP _md; + std::vector<SearchIterator::UP> _children; + fef::TermFieldMatchDataArray _childMatch; + std::vector<It> _iterators; + bool _strict; + + void unpack_children(uint32_t docid); + bool check_docid_match(uint32_t docid); + bool check_element_match(uint32_t docid); + +public: + SameElementSearch(fef::MatchData::UP md, + std::vector<SearchIterator::UP> children, + const fef::TermFieldMatchDataArray &childMatch, + bool strict); + void initRange(uint32_t begin_id, uint32_t end_id) override; + void doSeek(uint32_t docid) override; + void doUnpack(uint32_t) override {} + void visitMembers(vespalib::ObjectVisitor &visitor) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp index 1f5d090b914..3384e0fc8c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp @@ -118,7 +118,13 @@ SearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "docid", _docid); visit(visitor, "endid", _endid); } - + +const attribute::ISearchContext * +SearchIterator::getAttributeSearchContext() const +{ + return nullptr; +} + } // namespace queryeval } // namespace search diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index 63c2afd33aa..dfa342b018a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -12,6 +12,7 @@ namespace vespalib { class ObjectVisitor; } namespace search { class BitVector; } +namespace search::attribute { class ISearchContext; } namespace search::queryeval { @@ -338,6 +339,8 @@ public: virtual Trinary is_strict() const { return Trinary::Undefined; } + /** return the underlying attribute search context (or null if none available) */ + virtual const attribute::ISearchContext *getAttributeSearchContext() const; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp index 14a6cefaf1b..3829ea45e2b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp @@ -22,6 +22,7 @@ using search::query::Node; using search::query::ONear; using search::query::Or; using search::query::Phrase; +using search::query::SameElement; using search::query::PredicateQuery; using search::query::PrefixTerm; using search::query::QueryVisitor; @@ -84,6 +85,7 @@ struct TermAsStringVisitor : public QueryVisitor { void visit(ONear &) override {illegalVisit(); } void visit(Or &) override {illegalVisit(); } void visit(Phrase &) override {illegalVisit(); } + void visit(SameElement &) override {illegalVisit(); } void visit(Rank &) override {illegalVisit(); } void visit(WeakAnd &) override {illegalVisit(); } void visit(WeightedSetTerm &) override {illegalVisit(); } diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp index eac6f65a48e..b2920b39eaf 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp @@ -200,7 +200,7 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foobar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "index")); stack.AppendBuffer(&buf); keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); @@ -216,11 +216,11 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) // multiple phrase and term query stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "xyzzy")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "xyz")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2, "index")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foobar")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 3, "index")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "baz")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "zog")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 3)); @@ -241,7 +241,7 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "foo")); stack.Push(new search::ParseItem(search::ParseItem::ITEM_AND, 2)); stack.Push(new search::ParseItem(search::ParseItem::ITEM_TERM, "bar")); - stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2)); + stack.Push(new search::ParseItem(search::ParseItem::ITEM_PHRASE, 2, "index")); stack.AppendBuffer(&buf); keywords = _extractor->ExtractKeywords(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index e535eef660c..c7eb63a4480 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -280,6 +280,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const case search::ParseItem::ITEM_SUFFIXTERM: case search::ParseItem::ITEM_REGEXP: case search::ParseItem::ITEM_PREDICATE_QUERY: + case search::ParseItem::ITEM_SAME_ELEMENT: if (!v->VisitOther(&item, iterator.getArity())) { rc = SkipItem(&iterator); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index e153a898f6a..3a60db52cf3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -165,7 +165,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const break; case search::ParseItem::ITEM_PHRASE: - { + { // Must take the next arity TERMS and put together bool phraseterms_was_added = false; int phraseterms = si.getArity(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java new file mode 100644 index 00000000000..ec2702bcfaf --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ApplicationInstanceGenerator.java @@ -0,0 +1,13 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +/** + * @author hakon + */ +public interface ApplicationInstanceGenerator { + /** Make an ApplicationInstance based on current service status. */ + ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java new file mode 100644 index 00000000000..76ca59cf583 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGenerator.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class for generating an ApplicationInstance for the synthesized config server application. + * + * @author hakon + */ +public class ConfigServerAppGenerator implements ApplicationInstanceGenerator { + private final List<String> hostnames; + + public ConfigServerAppGenerator(List<String> hostnames) { + this.hostnames = hostnames; + } + + @Override + public ApplicationInstance makeApplicationInstance(ServiceStatusProvider statusProvider) { + Set<ServiceInstance> serviceInstances = hostnames.stream() + .map(hostname -> makeServiceInstance(hostname, statusProvider)) + .collect(Collectors.toSet()); + + ServiceCluster serviceCluster = new ServiceCluster( + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + serviceInstances); + + Set<ServiceCluster> serviceClusters = new HashSet<>(); + serviceClusters.add(serviceCluster); + + ApplicationInstance applicationInstance = new ApplicationInstance( + ConfigServerApplication.TENANT_ID, + ConfigServerApplication.APPLICATION_INSTANCE_ID, + serviceClusters); + + // Fill back-references + serviceCluster.setApplicationInstance(applicationInstance); + for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { + serviceInstance.setServiceCluster(serviceCluster); + } + + return applicationInstance; + } + + private ServiceInstance makeServiceInstance(String hostname, ServiceStatusProvider statusProvider) { + ConfigId configId = new ConfigId(ConfigServerApplication.CONFIG_ID_PREFIX + hostname); + ServiceStatus status = statusProvider.getStatus( + ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), + ConfigServerApplication.CLUSTER_ID, + ConfigServerApplication.SERVICE_TYPE, + configId); + + return new ServiceInstance(configId, new HostName(hostname), status); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java index 120a12609e1..132bb0927b8 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplication.java @@ -3,22 +3,11 @@ package com.yahoo.vespa.service.monitor.application; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * A service/application model of the config server with health status. */ @@ -36,34 +25,4 @@ public class ConfigServerApplication extends HostedVespaApplication { super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ClusterSpec.Id.from("zone-config-servers")); } - - public ApplicationInstance toApplicationInstance(List<String> hostnames) { - Set<ServiceInstance> serviceInstances = hostnames.stream() - .map(hostname -> new ServiceInstance( - new ConfigId(CONFIG_ID_PREFIX + hostname), - new HostName(hostname), - ServiceStatus.NOT_CHECKED)) - .collect(Collectors.toSet()); - - ServiceCluster serviceCluster = new ServiceCluster( - CLUSTER_ID, - SERVICE_TYPE, - serviceInstances); - - Set<ServiceCluster> serviceClusters = - Stream.of(serviceCluster).collect(Collectors.toSet()); - - ApplicationInstance applicationInstance = new ApplicationInstance( - TENANT_ID, - APPLICATION_INSTANCE_ID, - serviceClusters); - - // Fill back-references - serviceCluster.setApplicationInstance(applicationInstance); - for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { - serviceInstance.setServiceCluster(serviceCluster); - } - - return applicationInstance; - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java new file mode 100644 index 00000000000..2691a8bf1ee --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/application/DeployedAppGenerator.java @@ -0,0 +1,127 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.application; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceClusterKey; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class to generate an ApplicationInstance given service status for a standard (deployed) application. + * + * @author hakon + */ +public class DeployedAppGenerator implements ApplicationInstanceGenerator { + public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + + private final ApplicationInfo applicationInfo; + private final Zone zone; + + public DeployedAppGenerator(ApplicationInfo applicationInfo, Zone zone) { + this.applicationInfo = applicationInfo; + this.zone = zone; + } + + @Override + public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { + Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); + + for (HostInfo host : applicationInfo.getModel().getHosts()) { + HostName hostName = new HostName(host.getHostname()); + for (ServiceInfo serviceInfo : host.getServices()) { + ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); + ServiceInstance serviceInstance = + toServiceInstance( + applicationInfo.getApplicationId(), + serviceClusterKey.clusterId(), + serviceInfo, + hostName, + serviceStatusProvider); + + if (!groupedServiceInstances.containsKey(serviceClusterKey)) { + groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); + } + groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); + } + } + + Set<ServiceCluster> serviceClusters = groupedServiceInstances.entrySet().stream() + .map(entry -> new ServiceCluster( + entry.getKey().clusterId(), + entry.getKey().serviceType(), + entry.getValue())) + .collect(Collectors.toSet()); + + ApplicationInstance applicationInstance = new ApplicationInstance( + new TenantId(applicationInfo.getApplicationId().tenant().toString()), + toApplicationInstanceId(applicationInfo, zone), + serviceClusters); + + // Fill back-references + for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { + serviceCluster.setApplicationInstance(applicationInstance); + for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { + serviceInstance.setServiceCluster(serviceCluster); + } + } + + return applicationInstance; + } + + static ClusterId getClusterId(ServiceInfo serviceInfo) { + return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); + } + + private ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { + ClusterId clusterId = getClusterId(serviceInfo); + ServiceType serviceType = toServiceType(serviceInfo); + return new ServiceClusterKey(clusterId, serviceType); + } + + private ServiceInstance toServiceInstance( + ApplicationId applicationId, + ClusterId clusterId, + ServiceInfo serviceInfo, + HostName hostName, + ServiceStatusProvider serviceStatusProvider) { + ConfigId configId = new ConfigId(serviceInfo.getConfigId()); + + ServiceStatus status = serviceStatusProvider.getStatus( + applicationId, + clusterId, + toServiceType(serviceInfo), configId); + + return new ServiceInstance(configId, hostName, status); + } + + private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { + return new ApplicationInstanceId(String.format("%s:%s:%s:%s", + applicationInfo.getApplicationId().application().value(), + zone.environment().value(), + zone.region().value(), + applicationInfo.getApplicationId().instance().value())); + } + + private ServiceType toServiceType(ServiceInfo serviceInfo) { + return new ServiceType(serviceInfo.getServiceType()); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java index c4952979518..9da449289a7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ModelGenerator.java @@ -1,33 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor.internal; -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModel; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceCluster; -import com.yahoo.vespa.applicationmodel.ServiceClusterKey; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.application.ApplicationInstanceGenerator; +import com.yahoo.vespa.service.monitor.application.ConfigServerAppGenerator; +import com.yahoo.vespa.service.monitor.application.DeployedAppGenerator; -import java.util.HashMap; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -36,6 +24,16 @@ import java.util.stream.Collectors; public class ModelGenerator { public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + private final List<ApplicationInstanceGenerator> staticGenerators; + + public ModelGenerator(List<String> configServerHosts) { + if (configServerHosts.isEmpty()) { + staticGenerators = Collections.emptyList(); + } else { + staticGenerators = Collections.singletonList(new ConfigServerAppGenerator(configServerHosts)); + } + } + /** * Create service model based primarily on super model. * @@ -44,113 +42,15 @@ public class ModelGenerator { ServiceModel toServiceModel( SuperModel superModel, Zone zone, - List<String> configServerHosts, ServiceStatusProvider serviceStatusProvider) { - Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = new HashMap<>(); + List<ApplicationInstanceGenerator> generators = new ArrayList<>(staticGenerators); + superModel.getAllApplicationInfos() + .forEach(info -> generators.add(new DeployedAppGenerator(info, zone))); - for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) { - - ApplicationInstance applicationInstance = toApplicationInstance( - applicationInfo, - zone, - serviceStatusProvider); - applicationInstances.put(applicationInstance.reference(), applicationInstance); - } - - // The config server is part of the service model (but not super model) - if (!configServerHosts.isEmpty()) { - ConfigServerApplication configServerApplication = ConfigServerApplication.CONFIG_SERVER_APPLICATION; - ApplicationInstance configServerApplicationInstance = - configServerApplication.toApplicationInstance(configServerHosts); - applicationInstances.put(configServerApplicationInstance.reference(), configServerApplicationInstance); - } + Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = generators.stream() + .map(generator -> generator.makeApplicationInstance(serviceStatusProvider)) + .collect(Collectors.toMap(ApplicationInstance::reference, Function.identity())); return new ServiceModel(applicationInstances); } - - ApplicationInstance toApplicationInstance( - ApplicationInfo applicationInfo, - Zone zone, - ServiceStatusProvider serviceStatusProvider) { - Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); - - for (HostInfo host : applicationInfo.getModel().getHosts()) { - HostName hostName = new HostName(host.getHostname()); - for (ServiceInfo serviceInfo : host.getServices()) { - ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); - ServiceInstance serviceInstance = - toServiceInstance( - applicationInfo.getApplicationId(), - serviceClusterKey.clusterId(), - serviceInfo, - hostName, - serviceStatusProvider); - - if (!groupedServiceInstances.containsKey(serviceClusterKey)) { - groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); - } - groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); - } - } - - Set<ServiceCluster> serviceClusters = groupedServiceInstances.entrySet().stream() - .map(entry -> new ServiceCluster( - entry.getKey().clusterId(), - entry.getKey().serviceType(), - entry.getValue())) - .collect(Collectors.toSet()); - - ApplicationInstance applicationInstance = new ApplicationInstance( - new TenantId(applicationInfo.getApplicationId().tenant().toString()), - toApplicationInstanceId(applicationInfo, zone), - serviceClusters); - - // Fill back-references - for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { - serviceCluster.setApplicationInstance(applicationInstance); - for (ServiceInstance serviceInstance : serviceCluster.serviceInstances()) { - serviceInstance.setServiceCluster(serviceCluster); - } - } - - return applicationInstance; - } - - static ClusterId getClusterId(ServiceInfo serviceInfo) { - return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); - } - - private ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { - ClusterId clusterId = getClusterId(serviceInfo); - ServiceType serviceType = toServiceType(serviceInfo); - return new ServiceClusterKey(clusterId, serviceType); - } - - private ServiceInstance toServiceInstance( - ApplicationId applicationId, - ClusterId clusterId, - ServiceInfo serviceInfo, - HostName hostName, - ServiceStatusProvider serviceStatusProvider) { - ConfigId configId = new ConfigId(serviceInfo.getConfigId()); - - ServiceStatus status = serviceStatusProvider.getStatus( - applicationId, - clusterId, - toServiceType(serviceInfo), configId); - - return new ServiceInstance(configId, hostName, status); - } - - private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - return new ApplicationInstanceId(String.format("%s:%s:%s:%s", - applicationInfo.getApplicationId().application().value(), - zone.environment().value(), - zone.region().value(), - applicationInfo.getApplicationId().instance().value())); - } - - private ServiceType toServiceType(ServiceInfo serviceInfo) { - return new ServiceType(serviceInfo.getServiceType()); - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java index b2b6538fe6c..97c4fdda0f3 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/ServiceMonitorImpl.java @@ -11,6 +11,8 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import java.util.Collections; import java.util.List; @@ -28,7 +30,6 @@ public class ServiceMonitorImpl implements ServiceMonitor { Metric metric, Timer timer) { Zone zone = superModelProvider.getZone(); - List<String> configServerHosts = toConfigServerList(configserverConfig); ServiceMonitorMetrics metrics = new ServiceMonitorMetrics(metric, timer); UnionMonitorManager monitorManager = new UnionMonitorManager( @@ -39,9 +40,8 @@ public class ServiceMonitorImpl implements ServiceMonitor { SuperModelListenerImpl superModelListener = new SuperModelListenerImpl( monitorManager, metrics, - new ModelGenerator(), - zone, - configServerHosts); + new ModelGenerator(toConfigServerList(configserverConfig)), + zone); superModelListener.start(superModelProvider); serviceModelCache = new ServiceModelCache(superModelListener, timer); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java index 5e309d3c18d..b2f3617131b 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImpl.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.service.monitor.ServiceModel; -import java.util.List; import java.util.function.Supplier; import java.util.logging.Logger; @@ -19,24 +18,21 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv private final ServiceMonitorMetrics metrics; private final ModelGenerator modelGenerator; private final Zone zone; - private final List<String> configServerHosts; - // superModel and slobrokMonitorManager are always updated together + // superModel and monitorManager are always updated together // and atomically using this monitor. private final Object monitor = new Object(); - private final MonitorManager slobrokMonitorManager; + private final MonitorManager monitorManager; private SuperModel superModel; - SuperModelListenerImpl(MonitorManager slobrokMonitorManager, + SuperModelListenerImpl(MonitorManager monitorManager, ServiceMonitorMetrics metrics, ModelGenerator modelGenerator, - Zone zone, - List<String> configServerHosts) { - this.slobrokMonitorManager = slobrokMonitorManager; + Zone zone) { + this.monitorManager = monitorManager; this.metrics = metrics; this.modelGenerator = modelGenerator; this.zone = zone; - this.configServerHosts = configServerHosts; } void start(SuperModelProvider superModelProvider) { @@ -46,7 +42,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv // asynchronously even before snapshot() returns. this.superModel = superModelProvider.snapshot(this); superModel.getAllApplicationInfos().stream().forEach(application -> - slobrokMonitorManager.applicationActivated(superModel, application)); + monitorManager.applicationActivated(superModel, application)); } } @@ -54,7 +50,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv public void applicationActivated(SuperModel superModel, ApplicationInfo application) { synchronized (monitor) { this.superModel = superModel; - slobrokMonitorManager.applicationActivated(superModel, application); + monitorManager.applicationActivated(superModel, application); } } @@ -62,7 +58,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv public void applicationRemoved(SuperModel superModel, ApplicationId id) { synchronized (monitor) { this.superModel = superModel; - slobrokMonitorManager.applicationRemoved(superModel, id); + monitorManager.applicationRemoved(superModel, id); } } @@ -75,11 +71,7 @@ public class SuperModelListenerImpl implements SuperModelListener, Supplier<Serv dummy(measurement); // WARNING: The slobrok monitor manager may be out-of-sync with super model (no locking) - return modelGenerator.toServiceModel( - superModel, - zone, - configServerHosts, - slobrokMonitorManager); + return modelGenerator.toServiceModel(superModel, zone, monitorManager); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java index e224d6bfd12..82d2043bd17 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManager.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; import com.yahoo.vespa.service.monitor.application.ZoneApplication; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; /** * @author hakon @@ -32,6 +35,12 @@ public class UnionMonitorManager implements MonitorManager { ClusterId clusterId, ServiceType serviceType, ConfigId configId) { + + if (applicationId.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId())) { + // todo: use health + return ServiceStatus.NOT_CHECKED; + } + MonitorManager monitorManager = useHealth(applicationId, clusterId, serviceType) ? healthMonitorManager : slobrokMonitorManager; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java index 072886098d7..5a4b7251ae2 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/health/HealthMonitorManager.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +package com.yahoo.vespa.service.monitor.internal.health; import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; @@ -10,6 +10,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.monitor.application.ZoneApplication; +import com.yahoo.vespa.service.monitor.internal.MonitorManager; /** * @author hakon @@ -27,7 +28,10 @@ public class HealthMonitorManager implements MonitorManager { } @Override - public ServiceStatus getStatus(ApplicationId applicationId, ClusterId clusterId, ServiceType serviceType, ConfigId configId) { + public ServiceStatus getStatus(ApplicationId applicationId, + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { // TODO: Do proper health check if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { return ServiceStatus.UP; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitor.java index e0195e11759..a857be84cc7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitor.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImpl.java index 81173f9d835..aaaab22e742 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImpl.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.monitor.SlobrokApi; +import com.yahoo.vespa.service.monitor.internal.MonitorManager; import java.util.HashMap; import java.util.List; @@ -75,7 +76,8 @@ public class SlobrokMonitorManagerImpl implements SuperModelListener, SlobrokApi @Override public ServiceStatus getStatus(ApplicationId applicationId, - ClusterId clusterId, ServiceType serviceType, + ClusterId clusterId, + ServiceType serviceType, ConfigId configId) { Optional<String> slobrokServiceName = findSlobrokServiceName(serviceType, configId); if (slobrokServiceName.isPresent()) { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplicationTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGeneratorTest.java index 7fa6f82e183..58f99786017 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerApplicationTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/application/ConfigServerAppGeneratorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.service.monitor.application; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; @@ -11,8 +12,11 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -public class ConfigServerApplicationTest { +public class ConfigServerAppGeneratorTest { private static final String configServer1 = "cfg1.yahoo.com"; private static final String configServer2 = "cfg2.yahoo.com"; private static final String configServer3 = "cfg3.yahoo.com"; @@ -21,11 +25,13 @@ public class ConfigServerApplicationTest { configServer2, configServer3).collect(Collectors.toList()); + private final ServiceStatusProvider statusProvider = mock(ServiceStatusProvider.class); + @Test public void toApplicationInstance() throws Exception { - ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION; - ApplicationInstance applicationInstance = - application.toApplicationInstance(configServerList); + when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.NOT_CHECKED); + ApplicationInstance applicationInstance = new ConfigServerAppGenerator(configServerList) + .makeApplicationInstance(statusProvider); assertEquals( ConfigServerApplication.APPLICATION_INSTANCE_ID, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java index fca1512e3ea..186b22cf4ec 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ExampleModel.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitor; import java.util.ArrayList; import java.util.Arrays; @@ -51,13 +52,13 @@ public class ExampleModel { return new SuperModel(applicationInfos); } - static ApplicationBuilder createApplication(String tenant, - String applicationName) { + public static ApplicationBuilder createApplication(String tenant, + String applicationName) { return new ApplicationBuilder(tenant, applicationName); } - static class ApplicationBuilder { + public static class ApplicationBuilder { private final String tenant; private final String applicationName; private final List<ClusterBuilder> clusters = new ArrayList<>(); @@ -80,7 +81,7 @@ public class ExampleModel { hosts); } - ApplicationInfo build() { + public ApplicationInfo build() { List<String> allHosts = clusters.stream() .flatMap(clusterBuilder -> clusterBuilder.hosts.stream()) .distinct() diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java index 6e9fc25382a..a21691ee4d0 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/ModelGeneratorTest.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.application.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; import java.util.Collections; @@ -37,12 +38,12 @@ public class ModelGeneratorTest { public void toApplicationModelWithConfigServerApplication() throws Exception { SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(HOSTNAME, PORT); - ModelGenerator modelGenerator = new ModelGenerator(); - - Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); List<String> configServerHosts = Stream.of("cfg1", "cfg2", "cfg3") .collect(Collectors.toList()); + ModelGenerator modelGenerator = new ModelGenerator(configServerHosts); + + Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) @@ -52,7 +53,6 @@ public class ModelGeneratorTest { modelGenerator.toServiceModel( superModel, zone, - configServerHosts, slobrokMonitorManager); Map<ApplicationInstanceReference, @@ -82,12 +82,10 @@ public class ModelGeneratorTest { public void toApplicationModel() throws Exception { SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(HOSTNAME, PORT); - ModelGenerator modelGenerator = new ModelGenerator(); + ModelGenerator modelGenerator = new ModelGenerator(Collections.emptyList()); Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); - List<String> configServerHosts = Collections.emptyList(); - SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) .thenReturn(ServiceStatus.UP); @@ -96,7 +94,6 @@ public class ModelGeneratorTest { modelGenerator.toServiceModel( superModel, zone, - configServerHosts, slobrokMonitorManager); Map<ApplicationInstanceReference, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java index 6233f39b9cf..83bad0ddb2a 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SuperModelListenerImplTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,13 +24,11 @@ public class SuperModelListenerImplTest { ServiceMonitorMetrics metrics = mock(ServiceMonitorMetrics.class); ModelGenerator modelGenerator = mock(ModelGenerator.class); Zone zone = mock(Zone.class); - List<String> configServers = new ArrayList<>(); SuperModelListenerImpl listener = new SuperModelListenerImpl( slobrokMonitorManager, metrics, modelGenerator, - zone, - configServers); + zone); SuperModelProvider superModelProvider = mock(SuperModelProvider.class); SuperModel superModel = mock(SuperModel.class); @@ -47,6 +45,6 @@ public class SuperModelListenerImplTest { verify(slobrokMonitorManager).applicationActivated(superModel, application2); ServiceModel serviceModel = listener.get(); - verify(modelGenerator).toServiceModel(superModel, zone, configServers, slobrokMonitorManager); + verify(modelGenerator).toServiceModel(superModel, zone, slobrokMonitorManager); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java index e557f0451f4..b7c3ed8e1e1 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/UnionMonitorManagerTest.java @@ -6,6 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.monitor.internal.health.HealthMonitorManager; +import com.yahoo.vespa.service.monitor.internal.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; import static com.yahoo.vespa.applicationmodel.ClusterId.NODE_ADMIN; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImplTest.java index ab50b3192e3..8e4443df83b 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorManagerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorManagerImplTest.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorTest.java index 075647e9c16..5b230e81cf7 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/SlobrokMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/internal/slobrok/SlobrokMonitorTest.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.internal; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor.internal.slobrok; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.vespa.service.monitor.internal.ExampleModel; import org.junit.Test; import static org.mockito.Mockito.mock; diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp index 8fd96153dd7..ebeda4f1b8b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp @@ -56,7 +56,7 @@ JSONWriter::quote(const char * str, size_t len) case '\"': case '\\': v[j++] = '\\'; - //@fallthrough@ + [[fallthrough]]; default: v[j++] = str[i]; break; diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 0d66951f364..f31072aec6e 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -69,11 +69,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> @@ -101,40 +96,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> </plugin> - - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>compile</id> - <goals> - <goal>compile</goal> - </goals> - <phase>compile</phase> - </execution> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - <execution> - <phase>process-resources</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - <execution> - <phase>process-test-resources</phase> - <id>early-test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java new file mode 100644 index 00000000000..8d4126a01e3 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java @@ -0,0 +1,569 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl; + +import com.google.common.collect.Lists; +import com.yahoo.container.standalone.StandaloneContainerApplication; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.application.OsgiHeader; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; +import org.osgi.framework.BundleListener; +import org.osgi.framework.Filter; +import org.osgi.framework.FrameworkListener; +import org.osgi.framework.ServiceFactory; +import org.osgi.framework.ServiceListener; +import org.osgi.framework.ServiceObjects; +import org.osgi.framework.ServiceReference; +import org.osgi.framework.ServiceRegistration; +import org.osgi.framework.Version; +import org.osgi.framework.wiring.BundleCapability; +import org.osgi.framework.wiring.BundleRequirement; +import org.osgi.framework.wiring.BundleRevision; +import org.osgi.framework.wiring.BundleWire; +import org.osgi.framework.wiring.BundleWiring; +import org.osgi.resource.Capability; +import org.osgi.resource.Requirement; +import org.osgi.resource.Wire; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Dictionary; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.jar.Attributes; +import java.util.jar.JarFile; + +/** + * A (mock) OSGI implementation which loads classes from the system classpath + * + * @author Tony Vaagenes + * @author ollivir + */ +public final class ClassLoaderOsgiFramework implements OsgiFramework { + private BundleContextImpl bundleContextImpl = new BundleContextImpl(); + private SystemBundleImpl systemBundleImpl = new SystemBundleImpl(); + private BundleWiringImpl bundleWiringImpl = new BundleWiringImpl(); + + private List<URL> bundleLocations = new ArrayList<>(); + private List<Bundle> bundleList = Lists.newArrayList(systemBundleImpl); + private ClassLoader classLoader = null; + + private AtomicInteger nextBundleId = new AtomicInteger(1); + + @Override + public List<Bundle> installBundle(String bundleLocation) { + if (bundleLocation != null && bundleLocation.isEmpty() == false) { + try { + URL url = new URL(bundleLocation); + bundleLocations.add(url); + bundleList.add(new JarBundleImpl(url)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return bundles(); + } + + private ClassLoader getClassLoader() { + if (bundleLocations.isEmpty()) { + return getClass().getClassLoader(); + } else { + if (classLoader == null) { + classLoader = new URLClassLoader(bundleLocations.toArray(new URL[0]), getClass().getClassLoader()); + } + return classLoader; + } + } + + @Override + public void startBundles(List<Bundle> bundles, boolean privileged) { + } + + @Override + public void refreshPackages() { + } + + @Override + public BundleContext bundleContext() { + return bundleContextImpl; + } + + @Override + public List<Bundle> bundles() { + return bundleList; + } + + @Override + public void start() { + } + + @Override + public void stop() { + } + + private abstract class BundleImpl implements Bundle { + @Override + public int getState() { + return Bundle.ACTIVE; + } + + @Override + public void start(int options) { + } + + @Override + public void start() { + } + + @Override + public void stop(int options) { + } + + @Override + public void stop() { + } + + @Override + public void update(InputStream input) { + } + + @Override + public void update() { + } + + @Override + public void uninstall() { + } + + @Override + public Dictionary<String, String> getHeaders(String locale) { + return getHeaders(); + } + + @Override + public String getSymbolicName() { + return ClassLoaderOsgiFramework.this.getClass().getName(); + } + + @Override + public String getLocation() { + return getSymbolicName(); + } + + @Override + public ServiceReference<?>[] getRegisteredServices() { + return new ServiceReference<?>[0]; + } + + @Override + public ServiceReference<?>[] getServicesInUse() { + return getRegisteredServices(); + } + + @Override + public boolean hasPermission(Object permission) { + return true; + } + + @Override + public URL getResource(String name) { + return getClassLoader().getResource(name); + } + + @Override + public Class<?> loadClass(String name) throws ClassNotFoundException { + return getClassLoader().loadClass(name); + } + + @Override + public Enumeration<URL> getResources(String name) throws IOException { + return getClassLoader().getResources(name); + } + + @Override + public Enumeration<String> getEntryPaths(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public URL getEntry(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public Enumeration<URL> findEntries(String path, String filePattern, boolean recurse) { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastModified() { + return 1L; + } + + @Override + public BundleContext getBundleContext() { + throw new UnsupportedOperationException(); + } + + @Override + public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) { + return Collections.emptyMap(); + } + + @Override + @SuppressWarnings("unchecked") + public <T> T adapt(Class<T> clazz) { + if (clazz.equals(BundleRevision.class)) { + return (T) new BundleRevisionImpl(); + } else if (clazz.equals(BundleWiring.class)) { + return (T) new BundleWiringImpl(); + } else { + return null; + } + } + + @Override + public File getDataFile(String filename) { + return null; + } + + @Override + public int compareTo(Bundle o) { + return Long.compare(getBundleId(), o.getBundleId()); + } + } + + private class BundleRevisionImpl implements BundleRevision { + @Override + public String getSymbolicName() { + return this.getClass().getName(); + } + + @Override + public List<BundleRequirement> getDeclaredRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public Version getVersion() { + return Version.emptyVersion; + } + + @Override + public BundleWiring getWiring() { + return bundleWiringImpl; + } + + @Override + public List<BundleCapability> getDeclaredCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public int getTypes() { + return 0; + } + + @Override + public Bundle getBundle() { + throw new UnsupportedOperationException(); + } + + @Override + public List<Capability> getCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Requirement> getRequirements(String p1) { + throw new UnsupportedOperationException(); + } + } + + private class BundleWiringImpl implements BundleWiring { + @Override + public List<URL> findEntries(String p1, String p2, int p3) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getRequiredResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Capability> getResourceCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCurrent() { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getRequiredWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleCapability> getCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getProvidedResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getProvidedWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getRevision() { + throw new UnsupportedOperationException(); + } + + @Override + public List<Requirement> getResourceRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isInUse() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection<String> listResources(String p1, String p2, int p3) { + throw new UnsupportedOperationException(); + } + + @Override + public ClassLoader getClassLoader() { + return ClassLoaderOsgiFramework.this.getClassLoader(); + } + + @Override + public List<BundleRequirement> getRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getResource() { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle getBundle() { + throw new UnsupportedOperationException(); + } + } + + private class SystemBundleImpl extends BundleImpl { + @Override + public long getBundleId() { + return 0L; + } + + @Override + public Version getVersion() { + return Version.emptyVersion; + } + + @Override + public Dictionary<String, String> getHeaders() { + Hashtable<String, String> ret = new Hashtable<>(); + ret.put(OsgiHeader.APPLICATION, StandaloneContainerApplication.class.getName()); + return ret; + } + } + + private class JarBundleImpl extends BundleImpl { + private final long bundleId; + private final Dictionary<String, String> headers; + + JarBundleImpl(URL location) throws IOException { + this.bundleId = (long) nextBundleId.getAndIncrement(); + this.headers = retrieveHeaders(location); + } + + @Override + public long getBundleId() { + return bundleId; + } + + @Override + public Dictionary<String, String> getHeaders() { + return headers; + } + + @Override + public String getSymbolicName() { + return headers.get("Bundle-SymbolicName"); + } + + @Override + public Version getVersion() { + return Version.parseVersion(headers.get("Bundle-Version")); + } + + private Dictionary<String, String> retrieveHeaders(URL location) throws IOException { + try (JarFile jarFile = new JarFile(location.getFile())) { + Attributes attributes = jarFile.getManifest().getMainAttributes(); + Hashtable<String, String> ret = new Hashtable<>(); + attributes.forEach((k, v) -> ret.put(k.toString(), v.toString())); + return ret; + } + } + } + + private class BundleContextImpl implements BundleContext { + @Override + public String getProperty(String key) { + return null; + } + + @Override + public Bundle getBundle() { + return systemBundleImpl; + } + + @Override + public Bundle installBundle(String location, InputStream input) { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle installBundle(String location) { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle getBundle(long id) { + return systemBundleImpl; + } + + @Override + public Bundle[] getBundles() { + return new Bundle[] { systemBundleImpl }; + } + + @Override + public Bundle getBundle(String location) { + return systemBundleImpl; + } + + @Override + public void addServiceListener(ServiceListener listener, String filter) { + } + + @Override + public void addServiceListener(ServiceListener listener) { + } + + @Override + public void removeServiceListener(ServiceListener listener) { + } + + @Override + public void addBundleListener(BundleListener listener) { + } + + @Override + public void removeBundleListener(BundleListener listener) { + } + + @Override + public void addFrameworkListener(FrameworkListener listener) { + } + + @Override + public void removeFrameworkListener(FrameworkListener listener) { + } + + @Override + public ServiceRegistration<?> registerService(String[] classes, Object service, Dictionary<String, ?> properties) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceRegistration<?> registerService(String clazz, Object service, Dictionary<String, ?> properties) { + return null; + } + + @Override + public <S> ServiceRegistration<S> registerService(Class<S> clazz, S service, Dictionary<String, ?> properties) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?>[] getServiceReferences(String clazz, String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?>[] getAllServiceReferences(String clazz, String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?> getServiceReference(String clazz) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceReference<S> getServiceReference(Class<S> clazz) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> Collection<ServiceReference<S>> getServiceReferences(Class<S> clazz, String filter) { + return new ArrayList<>(); + } + + @Override + public <S> S getService(ServiceReference<S> reference) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean ungetService(ServiceReference<?> reference) { + throw new UnsupportedOperationException(); + } + + @Override + public File getDataFile(String filename) { + throw new UnsupportedOperationException(); + } + + @Override + public Filter createFilter(String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceRegistration<S> registerService(Class<S> aClass, ServiceFactory<S> serviceFactory, + Dictionary<String, ?> dictionary) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceObjects<S> getServiceObjects(ServiceReference<S> serviceReference) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java new file mode 100644 index 00000000000..a0fee3265df --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl; + +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneContainerRunner { + public static Path createApplicationPackage(String servicesXml) { + try { + return createApplicationDirectory(servicesXml); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Path createApplicationDirectory(String servicesXml) throws IOException { + Path applicationDir = Files.createTempDirectory("application"); + Path servicesXmlFile = applicationDir.resolve("services.xml"); + String content = servicesXml; + + if (!servicesXml.startsWith("<?xml")) { + content = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + servicesXml; + } + Files.write(servicesXmlFile, Utf8.toBytes(content)); + return applicationDir; + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java new file mode 100644 index 00000000000..4bbe9986d90 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java @@ -0,0 +1,97 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.net.HostName; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * FileAcquirer and FileRegistry working on a local directory. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class LocalFileDb implements FileAcquirer, FileRegistry { + private static final Constructor<FileReference> fileReferenceConstructor = createFileReferenceConstructor(); + + private final Map<FileReference, File> fileReferenceToFile = new HashMap<>(); + private final Path appPath; + + public LocalFileDb(Path appPath) { + this.appPath = appPath; + } + + /* FileAcquirer overrides */ + @Override + public File waitFor(FileReference reference, long l, TimeUnit timeUnit) { + synchronized (this) { + File file = fileReferenceToFile.get(reference); + if (file == null) { + throw new RuntimeException("Invalid file reference " + reference); + } + return file; + } + } + + @Override + public void shutdown() { + } + + /* FileRegistry overrides */ + public FileReference addFile(String relativePath) { + File file = appPath.resolve(relativePath).toFile(); + if (!file.exists()) { + throw new RuntimeException("The file does not exist: " + file.getPath()); + } + + FileReference fileReference = null; + try { + fileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Unable to create new FileReference", e); + } + fileReferenceToFile.put(fileReference, file); + return fileReference; + } + + @Override + public List<Entry> export() { + return fileReferenceToFile.entrySet().stream().map(entry -> new Entry(entry.getValue().getPath(), entry.getKey())) + .collect(Collectors.toList()); + } + + @Override + public FileReference addUri(String uri) { + throw new RuntimeException("addUri(String uri) is not implemented here."); + } + + public String fileSourceHost() { + return HostName.getLocalhost(); + } + + public Set<String> allRelativePaths() { + return fileReferenceToFile.values().stream().map(File::getPath).collect(Collectors.toSet()); + } + + private static Constructor<FileReference> createFileReferenceConstructor() { + try { + Constructor<FileReference> method = FileReference.class.getDeclaredConstructor(String.class); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ex) { + throw new IllegalStateException(ex); + } + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java new file mode 100644 index 00000000000..72937301954 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -0,0 +1,304 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.google.inject.AbstractModule; +import com.google.inject.ConfigurationException; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.ProvisionException; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.container.jdisc.ConfiguredApplication; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.application.Application; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import org.w3c.dom.Element; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.yahoo.collections.CollectionUtil.first; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class StandaloneContainerApplication implements Application { + public static final String PACKAGE_NAME = "standalone_jdisc_container"; + public static final String APPLICATION_LOCATION_INSTALL_VARIABLE = PACKAGE_NAME + ".app_location"; + public static final String DEPLOYMENT_PROFILE_INSTALL_VARIABLE = PACKAGE_NAME + ".deployment_profile"; + public static final String DISABLE_NETWORKING_ANNOTATION = "JDisc.disableNetworking"; + public static final Named APPLICATION_PATH_NAME = Names.named(APPLICATION_LOCATION_INSTALL_VARIABLE); + public static final Named CONFIG_MODEL_REPO_NAME = Names.named("ConfigModelRepo"); + + private static final String DEFAULT_TMP_BASE_DIR = Defaults.getDefaults().underVespaHome("tmp"); + private static final String TMP_DIR_NAME = "standalone_container"; + + private static final StaticConfigDefinitionRepo configDefinitionRepo = new StaticConfigDefinitionRepo(); + + private final Injector injector; + private final Path applicationPath; + private final LocalFileDb distributedFiles; + private final ConfigModelRepo configModelRepo; + private final Networking networkingOption; + private final VespaModel modelRoot; + private final Application configuredApplication; + private final Container container; + + @Inject + public StandaloneContainerApplication(Injector injector) { + this.injector = injector; + ConfiguredApplication.ensureVespaLoggingInitialized(); + this.applicationPath = injectedApplicationPath().orElseGet(this::installApplicationPath); + this.distributedFiles = new LocalFileDb(applicationPath); + this.configModelRepo = resolveConfigModelRepo(); + this.networkingOption = resolveNetworkingOption(); + + try { + Pair<VespaModel, Container> tpl = withTempDir(preprocessedApplicationDir -> createContainerModel(applicationPath, + distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo)); + this.modelRoot = tpl.getFirst(); + this.container = tpl.getSecond(); + } catch (RuntimeException r) { + throw r; + } catch (Exception e) { + throw new RuntimeException("Failed to create ContainerModel", e); + } + this.configuredApplication = createConfiguredApplication(container); + } + + private ConfigModelRepo resolveConfigModelRepo() { + try { + return injector.getInstance(Key.get(ConfigModelRepo.class, CONFIG_MODEL_REPO_NAME)); + } catch (Exception e) { + return new ConfigModelRepo(); + } + } + + private Networking resolveNetworkingOption() { + try { + Boolean networkingDisable = injector.getInstance(Key.get(Boolean.class, Names.named(DISABLE_NETWORKING_ANNOTATION))); + if (networkingDisable != null) { + return networkingDisable ? Networking.disable : Networking.enable; + } + } catch (Exception ignored) { + } + return Networking.enable; + } + + private Application createConfiguredApplication(Container container) { + Injector augmentedInjector = injector.createChildInjector(new AbstractModule() { + @Override + public void configure() { + bind(SubscriberFactory.class).toInstance(new StandaloneSubscriberFactory(modelRoot)); + } + }); + + System.setProperty("config.id", container.getConfigId()); + return augmentedInjector.getInstance(ConfiguredApplication.class); + } + + private Optional<Path> injectedApplicationPath() { + try { + return Optional.ofNullable(injector.getInstance(Key.get(Path.class, APPLICATION_PATH_NAME))); + } catch (ConfigurationException | ProvisionException ignored) { + } + return Optional.empty(); + } + + private Path installApplicationPath() { + Optional<String> variable = optionalInstallVariable(APPLICATION_LOCATION_INSTALL_VARIABLE); + + return variable.map(Paths::get) + .orElseThrow(() -> new IllegalStateException("Environment variable not set: " + APPLICATION_LOCATION_INSTALL_VARIABLE)); + } + + @Override + public void start() { + try { + com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles); + configuredApplication.start(); + } catch (Exception e) { + com.yahoo.container.Container.resetInstance(); + throw e; + } + } + + @Override + public void stop() { + configuredApplication.stop(); + } + + @Override + public void destroy() { + com.yahoo.container.Container.resetInstance(); + configuredApplication.destroy(); + } + + public Container container() { + return container; + } + + private interface ThrowingFunction<T, U> { + U apply(T input) throws Exception; + } + + private static <T> T withTempDir(ThrowingFunction<File, T> f) throws Exception { + File tmpDir = createTempDir(); + try { + return f.apply(tmpDir); + } finally { + IOUtils.recursiveDeleteDir(tmpDir); + } + } + + private static File createTempDir() { + Path basePath; + if (new File(DEFAULT_TMP_BASE_DIR).exists()) { + basePath = Paths.get(DEFAULT_TMP_BASE_DIR); + } else { + basePath = Paths.get(System.getProperty("java.io.tmpdir")); + } + + try { + Path tmpDir = Files.createTempDirectory(basePath, TMP_DIR_NAME); + return tmpDir.toFile(); + } catch (IOException e) { + throw new RuntimeException("Cannot create temp directory", e); + } + } + + private static void validateApplication(ApplicationPackage applicationPackage) { + try { + applicationPackage.validateXML(); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) { + Optional<String> profile = optionalInstallVariable(DEPLOYMENT_PROFILE_INSTALL_VARIABLE); + if (profile.isPresent()) { + String profileName = profile.get(); + if ("configserver".equals(profileName)) { + return new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()); + } else { + throw new RuntimeException("Invalid deployment profile '" + profileName + "'"); + } + } else { + return new ContainerModelBuilder(true, networkingOption); + } + } + + static Pair<VespaModel, Container> createContainerModel(Path applicationPath, FileRegistry fileRegistry, + File preprocessedApplicationDir, Networking networkingOption, ConfigModelRepo configModelRepo) throws Exception { + DeployLogger logger = new BaseDeployLogger(); + FilesApplicationPackage rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile()) + .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build(); + ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger); + validateApplication(applicationPackage); + DeployState deployState = new DeployState.Builder().applicationPackage(applicationPackage).fileRegistry(fileRegistry) + .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(true); + + VespaModel root = VespaModel.createIncomplete(deployState); + ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(), + deployState.getProperties().vespaVersion(), deployState.getProperties().applicationId()); + + Element spec = containerRootElement(applicationPackage); + ContainerModel containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec); + containerModel.getCluster().prepare(); + initializeContainerModel(containerModel, configModelRepo); + Container container = first(containerModel.getCluster().getContainers()); + + // TODO: Separate out model finalization from the VespaModel constructor, + // such that the above and below code to finalize the container can be + // replaced by root.finalize(); + + initializeContainer(container, spec); + + root.freezeModelTopology(); + return new Pair<>(root, container); + } + + private static void initializeContainer(Container container, Element spec) { + HostResource host = container.getRoot().getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + + container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)); + container.setHostResource(host); + container.initService(); + } + + private static Element getJDiscInServices(Element element) { + List<Element> jDiscElements = new ArrayList<>(); + for (ConfigModelId cid : ContainerModelBuilder.configModelIds) { + List<Element> children = XML.getChildren(element, cid.getName()); + jDiscElements.addAll(children); + } + + if (jDiscElements.size() == 1) { + return jDiscElements.get(0); + } else if (jDiscElements.isEmpty()) { + throw new RuntimeException("No jdisc element found under services."); + } else { + List<String> nameAndId = jDiscElements.stream().map(e -> e.getNodeName() + " id='" + e.getAttribute("id") + "'") + .collect(Collectors.toList()); + throw new RuntimeException("Found multiple JDisc elements: " + String.join(", ", nameAndId)); + } + } + + private static Element containerRootElement(ApplicationPackage applicationPackage) { + Element element = XmlHelper.getDocument(applicationPackage.getServices()).getDocumentElement(); + String nodeName = element.getNodeName(); + + if (ContainerModelBuilder.configModelIds.stream().anyMatch(id -> id.getName().equals(nodeName))) { + return element; + } else { + return getJDiscInServices(element); + } + } + + @SuppressWarnings("deprecation") // TODO: what is the not-deprecated way? + private static void initializeContainerModel(ContainerModel containerModel, ConfigModelRepo configModelRepo) { + containerModel.initialize(configModelRepo); + } + + private static Optional<String> optionalInstallVariable(String name) { + Optional<String> fromEnv = Optional.ofNullable(System.getenv((name.replace(".", "__")))); + if (fromEnv.isPresent()) { + return fromEnv; + } + return Optional.ofNullable(System.getProperty(name)); // for unit testing + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java new file mode 100644 index 00000000000..6ea2671b05b --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java @@ -0,0 +1,131 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.model.VespaModel; + +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class StandaloneSubscriberFactory implements SubscriberFactory { + private final VespaModel root; + + public StandaloneSubscriberFactory(VespaModel root) { + this.root = root; + } + + private class StandaloneSubscriber implements Subscriber { + + private final Set<ConfigKey<ConfigInstance>> configKeys; + private long generation = -1L; + + StandaloneSubscriber(Set<ConfigKey<ConfigInstance>> configKeys) { + this.configKeys = configKeys; + } + + @Override + public boolean internalRedeploy() { return false; } + + @Override + public boolean configChanged() { + return generation == 0; + } + + @Override + public void close() { + } + + @Override + public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() { + Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>(); + for (ConfigKey<ConfigInstance> key : configKeys) { + ConfigInstance.Builder builder = root.getConfig(newBuilderInstance(key), key.getConfigId()); + if (builder == null) { + throw new RuntimeException("Invalid config id " + key.getConfigId()); + } + ret.put(key, newConfigInstance(builder)); + } + return ret; + } + + @Override + public long waitNextGeneration() { + generation++; + + if (generation != 0) { + try { + while (!Thread.interrupted()) { + Thread.sleep(10000); + } + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + } + + return generation; + } + + // if waitNextGeneration has not yet been called, -1 should be returned + @Override + public long generation() { + return generation; + } + } + + @Override + @SuppressWarnings("unchecked") + public Subscriber getSubscriber(Set<? extends ConfigKey<?>> configKeys) { + return new StandaloneSubscriber((Set<ConfigKey<ConfigInstance>>) configKeys); + } + + public void reloadActiveSubscribers(long generation) { + throw new RuntimeException("unsupported"); + } + + private static ConfigInstance.Builder newBuilderInstance(ConfigKey<ConfigInstance> key) { + try { + return builderClass(key).getDeclaredConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException("ConfigInstance builder cannot be instantiated", e); + } + } + + @SuppressWarnings("unchecked") + private static Class<ConfigInstance.Builder> builderClass(ConfigKey<ConfigInstance> key) { + Class<?> configClass = key.getConfigClass(); + if (configClass != null) { + Class<?>[] nestedClasses = configClass.getClasses(); + for (Class<?> clazz : nestedClasses) { + if (clazz.getName().equals(key.getConfigClass().getName() + "$Builder")) { + return (Class<ConfigInstance.Builder>) clazz; + } + } + } + throw new RuntimeException("Builder class for " + (configClass == null ? null : configClass.getName()) + " could not be located"); + } + + private static ConfigInstance newConfigInstance(ConfigBuilder builder) { + try { + return configClass(builder).getConstructor(builder.getClass()).newInstance(builder); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException("ConfigInstance cannot be instantiated", e); + } + } + + @SuppressWarnings("unchecked") + private static Class<ConfigInstance> configClass(ConfigBuilder builder) { + return (Class<ConfigInstance>) builder.getClass().getEnclosingClass(); + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala deleted file mode 100644 index ac8636de2cb..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.impl - -import java.io.InputStream -import java.net.{URL, URLClassLoader} -import java.util -import java.util.concurrent.atomic.AtomicInteger -import java.util.jar.JarFile -import java.util.{Collections, Dictionary, Hashtable} - -import com.yahoo.container.standalone.StandaloneContainerApplication -import com.yahoo.jdisc.application.{OsgiFramework, OsgiHeader} -import org.osgi.framework._ -import org.osgi.framework.wiring._ -import org.osgi.resource.{Capability, Requirement, Wire} - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -/** - * A (mock) OSGI implementation which loads classes from the system classpath - * - * @author tonytv - */ -final class ClassLoaderOsgiFramework extends OsgiFramework { - private val bundleLocations = new ArrayBuffer[URL] - private val bundleList = ArrayBuffer[Bundle](SystemBundleImpl) - private var classLoader: ClassLoader = null - - private val nextBundleId = new AtomicInteger(1) - - override def installBundle(bundleLocation: String) = { - if (bundleLocation != "") { - val url = new URL(bundleLocation) - bundleLocations += url - bundleList += new JarBundleImpl(url) - } - - bundles() - } - - def getClassLoader = { - if (bundleLocations.isEmpty) { - getClass.getClassLoader - } else { - if(classLoader == null) - classLoader = new URLClassLoader(bundleLocations.toArray, getClass.getClassLoader) - - classLoader - } - } - - override def startBundles(bundles: util.List[Bundle], privileged: Boolean) {} - - override def refreshPackages() {} - - override def bundleContext():BundleContext = BundleContextImpl - - override def bundles() = bundleList.asJava - - override def start() {} - - override def stop() {} - - private abstract class BundleImpl extends Bundle { - override def getState = Bundle.ACTIVE - - override def start(options: Int) {} - override def start() {} - override def stop(options: Int) {} - override def stop() {} - override def update(input: InputStream) {} - override def update() {} - override def uninstall() {} - - override def getHeaders(locale: String) = getHeaders - - override def getSymbolicName = ClassLoaderOsgiFramework.this.getClass.getName - override def getLocation = getSymbolicName - - override def getRegisteredServices = Array[ServiceReference[_]]() - override def getServicesInUse = getRegisteredServices - - override def hasPermission(permission: Any) = true - - override def getResource(name: String) = getClassLoader.getResource(name) - override def loadClass(name: String) = getClassLoader.loadClass(name) - override def getResources(name: String) = getClassLoader.getResources(name) - - override def getEntryPaths(path: String) = throw new UnsupportedOperationException - override def getEntry(path: String) = throw new UnsupportedOperationException - override def findEntries(path: String, filePattern: String, recurse: Boolean) = throw new UnsupportedOperationException - - override def getLastModified = 1L - - override def getBundleContext = throw new UnsupportedOperationException - override def getSignerCertificates(signersType: Int) = Collections.emptyMap() - - override def adapt[A](`type`: Class[A]): A = { - if (`type` == classOf[BundleRevision]) BundleRevisionImpl.asInstanceOf[A] - else if (`type` == classOf[BundleWiring]) BundleWiringImpl.asInstanceOf[A] - else null.asInstanceOf[A] - } - - override def getDataFile(filename: String) = null - override def compareTo(o: Bundle) = getBundleId compareTo o.getBundleId - } - - private object BundleRevisionImpl extends BundleRevision { - override def getSymbolicName: String = this.getClass.getName - override def getDeclaredRequirements(p1: String): util.List[BundleRequirement] = throw new UnsupportedOperationException - override def getVersion: Version = Version.emptyVersion - override def getWiring: BundleWiring = BundleWiringImpl - override def getDeclaredCapabilities(p1: String): util.List[BundleCapability] = throw new UnsupportedOperationException - override def getTypes: Int = 0 - override def getBundle: Bundle = throw new UnsupportedOperationException - override def getCapabilities(p1: String): util.List[Capability] = throw new UnsupportedOperationException - override def getRequirements(p1: String): util.List[Requirement] = throw new UnsupportedOperationException - } - - private object BundleWiringImpl extends BundleWiring { - override def findEntries(p1: String, p2: String, p3: Int): util.List[URL] = ??? - override def getRequiredResourceWires(p1: String): util.List[Wire] = ??? - override def getResourceCapabilities(p1: String): util.List[Capability] = ??? - override def isCurrent: Boolean = ??? - override def getRequiredWires(p1: String): util.List[BundleWire] = ??? - override def getCapabilities(p1: String): util.List[BundleCapability] = ??? - override def getProvidedResourceWires(p1: String): util.List[Wire] = ??? - override def getProvidedWires(p1: String): util.List[BundleWire] = ??? - override def getRevision: BundleRevision = ??? - override def getResourceRequirements(p1: String): util.List[Requirement] = ??? - override def isInUse: Boolean = ??? - override def listResources(p1: String, p2: String, p3: Int): util.Collection[String] = ??? - override def getClassLoader: ClassLoader = ClassLoaderOsgiFramework.this.getClassLoader - override def getRequirements(p1: String): util.List[BundleRequirement] = ??? - override def getResource: BundleRevision = ??? - override def getBundle: Bundle = ??? - } - - private object SystemBundleImpl extends BundleImpl { - override val getBundleId = 0L - override def getVersion = Version.emptyVersion - override def getHeaders: Dictionary[String, String] = new Hashtable[String, String](Map(OsgiHeader.APPLICATION -> classOf[StandaloneContainerApplication].getName).asJava) - } - - - private class JarBundleImpl(location: URL) extends BundleImpl { - override val getBundleId = nextBundleId.getAndIncrement.asInstanceOf[Long] - - private val headers = retrieveHeaders(location) - - override def getHeaders: Dictionary[String, String] = headers - override val getSymbolicName = headers.get("Bundle-SymbolicName") - override val getVersion = Version.parseVersion(headers.get("Bundle-Version")) - - - private def retrieveHeaders(location: URL) = { - val jarFile = new JarFile(location.getFile) - try { - val attributes = jarFile.getManifest.getMainAttributes - new Hashtable[String, String](attributes.entrySet().asScala.map( entry => entry.getKey.toString -> entry.getValue.toString).toMap.asJava) - } finally { - jarFile.close() - } - } - } - - private object BundleContextImpl extends BundleContext { - private val bundleImpl = SystemBundleImpl - - override def getProperty(key: String) = null - override def getBundle = bundleImpl - override def installBundle(location: String, input: InputStream) = throw new UnsupportedOperationException - override def installBundle(location: String) = throw new UnsupportedOperationException - - override def getBundle(id: Long) = bundleImpl - override def getBundles = Array(bundleImpl) - override def getBundle(location: String) = bundleImpl - - override def addServiceListener(listener: ServiceListener, filter: String) {} - override def addServiceListener(listener: ServiceListener) {} - override def removeServiceListener(listener: ServiceListener) {} - override def addBundleListener(listener: BundleListener) {} - override def removeBundleListener(listener: BundleListener) {} - - override def addFrameworkListener(listener: FrameworkListener) {} - override def removeFrameworkListener(listener: FrameworkListener) {} - - override def registerService(clazzes: Array[String], service: Any, properties: Dictionary[String, _]) = throw new UnsupportedOperationException - override def registerService(clazz: String, service: Any, properties: Dictionary[String, _]) = null - override def registerService[S](clazz: Class[S], service: S, properties: Dictionary[String, _]) = throw new UnsupportedOperationException - override def getServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException - override def getAllServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException - override def getServiceReference(clazz: String) = throw new UnsupportedOperationException - override def getServiceReference[S](clazz: Class[S]) = throw new UnsupportedOperationException - override def getServiceReferences[S](clazz: Class[S], filter: String) = Collections.emptyList() - override def getService[S](reference: ServiceReference[S]) = throw new UnsupportedOperationException - override def ungetService(reference: ServiceReference[_]) = throw new UnsupportedOperationException - override def getDataFile(filename: String) = throw new UnsupportedOperationException - override def createFilter(filter: String) = throw new UnsupportedOperationException - - override def registerService[S](aClass: Class[S], serviceFactory: ServiceFactory[S], dictionary: Dictionary[String, _]): ServiceRegistration[S] = throw new UnsupportedOperationException - override def getServiceObjects[S](serviceReference: ServiceReference[S]): ServiceObjects[S] = throw new UnsupportedOperationException - } - -} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala deleted file mode 100644 index 91634250fc5..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.impl - -import java.nio.file.Files -import com.yahoo.text.Utf8 - -/** - * @author tonytv - */ -final class StandaloneContainerRunner { - - - -} - -object StandaloneContainerRunner { - def createApplicationPackage(servicesXml: String) = { - val applicationDir = Files.createTempDirectory("application") - - val servicesXmlFile = applicationDir.resolve("services.xml"); - var content = servicesXml; - if ( ! servicesXml.startsWith("<?xml")) - content = """<?xml version="1.0" encoding="utf-8" ?>""" + '\n' + servicesXml - Files.write(servicesXmlFile, Utf8.toBytes(content)) - applicationDir - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala deleted file mode 100644 index 61128347319..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -/** - * @author tonytv - */ -trait Converter[T] { - def convert(s: String): T -} - -object Converter { - def toConverter[T](f: String => T) = new Converter[T] { - override def convert(s: String) = f(s) - } - - implicit val intConverter = toConverter(_.toInt) - implicit val longConverter = toConverter(_.toLong) - implicit val boolConverter = toConverter(_.toBoolean) - implicit val stringConverter = toConverter(identity) - - implicit val javaIntegerConverter:Converter[Integer] = toConverter(_.toInt) - implicit val javaLongConverter:Converter[java.lang.Long] = toConverter(_.toLong) - implicit val javaBooleanConverter:Converter[java.lang.Boolean] = toConverter(_.toBoolean) - - -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala deleted file mode 100644 index 2aab88d8319..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -/** - * @author tonytv - * TODO: copied from standalone-container. Move to separate lib module instead. - */ -object Environment { - def optionalInstallVariable(name: String) = { - env(name.replace(".", "__")). - orElse(systemProperty(name)) //for unit testing - } - - def installVariable(name: String) = { - optionalInstallVariable(name). - getOrElse { - throw new IllegalStateException("Environment variable not set: " + name) - } - } - - def env(name: String) = Option(System.getenv(name)) - def systemProperty(name: String) = Option(System.getProperty(name)) -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala deleted file mode 100644 index 6507b4c72f0..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.io.File -import java.lang.reflect.Constructor -import java.nio.file.Path -import java.util -import java.util.concurrent.TimeUnit - -import com.yahoo.config.FileReference -import com.yahoo.config.application.api.FileRegistry -import com.yahoo.config.application.api.FileRegistry.Entry -import com.yahoo.container.standalone.LocalFileDb._ -import com.yahoo.filedistribution.fileacquirer.FileAcquirer -import com.yahoo.net.HostName - -import scala.collection.JavaConverters._ -import scala.collection.mutable - - -/** - * FileAcquirer and FileRegistry working on a local directory. - * @author tonytv - */ -class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry { - private val fileReferenceToFile = mutable.Map[FileReference, File]() - - /** *** FileAcquirer overrides *****/ - def waitFor(reference: FileReference, l: Long, timeUnit: TimeUnit): File = { - synchronized { - fileReferenceToFile.get(reference).getOrElse { - throw new RuntimeException("Invalid file reference " + reference) - } - } - } - - override def shutdown() {} - - /** *** FileRegistry overrides *****/ - def addFile(relativePath: String): FileReference = { - val file = appPath.resolve(relativePath).toFile - if (!file.exists) { - throw new RuntimeException("The file does not exist: " + file.getPath) - } - - val fileReference: FileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath) - fileReferenceToFile.put(fileReference, file) - fileReference - } - - def fileSourceHost: String = - HostName.getLocalhost - - def allRelativePaths: java.util.Set[String] = { - new java.util.HashSet(fileReferenceToFile.values.map(_.getPath).asJavaCollection) - } - - override def export(): util.List[Entry] = { - new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}.asJavaCollection) - } - - override def addUri(uri: String): FileReference = { - throw new RuntimeException("addUri(uri: String) is not implemented here."); - } -} - -object LocalFileDb { - private def createFileReferenceConstructor: Constructor[FileReference] = { - val method: Constructor[FileReference] = classOf[FileReference].getDeclaredConstructor(classOf[String]) - method.setAccessible(true) - method - } - - private val fileReferenceConstructor: Constructor[FileReference] = createFileReferenceConstructor -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala deleted file mode 100644 index 5271cd400d4..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.io.{File, IOException} -import java.lang.{Boolean => JBoolean} -import java.nio.file.{FileSystems, Files, Path, Paths} - -import com.google.inject.name.Names -import com.google.inject.{AbstractModule, Inject, Injector, Key} -import com.yahoo.collections.CollectionUtil.first -import com.yahoo.config.application.api.{ApplicationPackage, FileRegistry} -import com.yahoo.config.model.application.provider._ -import com.yahoo.config.model.builder.xml.XmlHelper -import com.yahoo.config.model.deploy.DeployState -import com.yahoo.config.model.{ApplicationConfigProducerRoot, ConfigModelRepo} -import com.yahoo.config.provision.Zone -import com.yahoo.container.di.config.SubscriberFactory -import com.yahoo.container.jdisc.ConfiguredApplication -import com.yahoo.container.standalone.Environment._ -import com.yahoo.container.standalone.StandaloneContainerApplication._ -import com.yahoo.io.IOUtils -import com.yahoo.jdisc.application.Application -import com.yahoo.text.XML -import com.yahoo.vespa.defaults.Defaults -import com.yahoo.vespa.model.VespaModel -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder -import com.yahoo.vespa.model.container.{Container, ContainerModel} -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking -import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ContainerModelBuilder} -import org.w3c.dom.Element - -import scala.collection.JavaConverters._ -import scala.util.Try - -/** - * @author Tony Vaagenes - * @author gjoranv - */ -class StandaloneContainerApplication @Inject()(injector: Injector) extends Application { - - ConfiguredApplication.ensureVespaLoggingInitialized() - - val applicationPath: Path = injectedApplicationPath.getOrElse(installApplicationPath) - - val distributedFiles = new LocalFileDb(applicationPath) - - val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo) - - val networkingOption = Try { - injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation))) - }.map { - case JBoolean.TRUE => Networking.disable - case JBoolean.FALSE => Networking.enable - }.getOrElse(Networking.enable) - - val (modelRoot, container) = withTempDir( - preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo)) - - val configuredApplication = createConfiguredApplication(container) - - def createConfiguredApplication(container: Container): Application = { - val augmentedInjector = injector.createChildInjector(new AbstractModule { - def configure() { - bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot)) - } - }) - - System.setProperty("config.id", container.getConfigId) //TODO: DRY - augmentedInjector.getInstance(classOf[ConfiguredApplication]) - } - - def injectedApplicationPath = Try { - injector.getInstance(Key.get(classOf[Path], applicationPathName)) - }.toOption - - def installApplicationPath = path(installVariable(applicationLocationInstallVariable)) - - override def start() { - try { - com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles) - configuredApplication.start() - } - catch { - case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; } - } - } - - override def stop() { - configuredApplication.stop() - } - - override def destroy() { - com.yahoo.container.Container.resetInstance() - configuredApplication.destroy() - } -} - -object StandaloneContainerApplication { - val packageName = "standalone_jdisc_container" - val applicationLocationInstallVariable = s"$packageName.app_location" - val deploymentProfileInstallVariable = s"$packageName.deployment_profile" - - val applicationPathName = Names.named(applicationLocationInstallVariable) - - val disableNetworkingAnnotation = "JDisc.disableNetworking" - val configModelRepoName = Names.named("ConfigModelRepo") - val configDefinitionRepo = new StaticConfigDefinitionRepo() - - val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp") - val tmpDirName = "standalone_container" - - private def withTempDir[T](f: File => T): T = { - val tmpDir = createTempDir() - try { - f(tmpDir) - } finally { - IOUtils.recursiveDeleteDir(tmpDir) - } - } - - private def createTempDir(): File = { - def getBaseDir: Path = { - val tmpBaseDir = - if (new File(defaultTmpBaseDir).exists()) - defaultTmpBaseDir - else - System.getProperty("java.io.tmpdir") - - Paths.get(tmpBaseDir) - } - - val basePath: Path = getBaseDir - val tmpDir = Files.createTempDirectory(basePath, tmpDirName) - tmpDir.toFile - } - - private def validateApplication(applicationPackage: ApplicationPackage) = { - try { - applicationPackage.validateXML() - } catch { - case e: IOException => throw new IllegalArgumentException(e) - } - } - - def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { - optionalInstallVariable(deploymentProfileInstallVariable) match { - case None => new ContainerModelBuilder(true, networkingOption) - case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables) - case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") - } - } - - def createContainerModel(applicationPath: Path, - fileRegistry: FileRegistry, - preprocessedApplicationDir: File, - networkingOption: Networking, - configModelRepo: ConfigModelRepo = new ConfigModelRepo): (VespaModel, Container) = { - val logger = new BaseDeployLogger - val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build() - val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger) - validateApplication(applicationPackage) - val deployState = new DeployState.Builder(). - applicationPackage(applicationPackage). - fileRegistry(fileRegistry). - deployLogger(logger). - configDefinitionRepo(configDefinitionRepo). - build(true) - - val root = VespaModel.createIncomplete(deployState) - val vespaRoot = new ApplicationConfigProducerRoot(root, - "vespa", - deployState.getDocumentModel, - deployState.getProperties.vespaVersion(), - deployState.getProperties.applicationId()) - - val spec = containerRootElement(applicationPackage) - val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec) - containerModel.getCluster().prepare() - DeprecationSuppressor.initializeContainerModel(containerModel, configModelRepo) - val container = first(containerModel.getCluster().getContainers) - - // TODO: Separate out model finalization from the VespaModel constructor, - // such that the above and below code to finalize the container can be - // replaced by root.finalize(); - - initializeContainer(container, spec) - - root.freezeModelTopology() - (root, container) - } - - def initializeContainer(container: Container, spec: Element) { - val host = container.getRoot.getHostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC) - - container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)) - container.setHostResource(host) - container.initService() - } - - def getJDiscInServices(element: Element): Element = { - def nameAndId(elements: List[Element]): List[String] = { - elements map { e => s"${e.getNodeName} id='${e.getAttribute("id")}'" } - } - - val jDiscElements = ContainerModelBuilder.configModelIds.asScala flatMap { name => XML.getChildren(element, name.getName).asScala } - jDiscElements.toList match { - case List(e) => e - case Nil => throw new RuntimeException("No jdisc element found under services.") - case multipleElements: List[Element] => throw new RuntimeException("Found multiple JDisc elements: " + nameAndId(multipleElements).mkString(", ")) - } - } - - def containerRootElement(applicationPackage: ApplicationPackage) : Element = { - val element = XmlHelper.getDocument(applicationPackage.getServices).getDocumentElement - val nodeName = element.getNodeName - - if (ContainerModelBuilder.configModelIds.asScala.map(_.getName).contains(nodeName)) element - else getJDiscInServices(element) - } - - def path(s: String) = FileSystems.getDefault.getPath(s) - - // Ugly hack required since Scala cannot suppress warnings. https://issues.scala-lang.org/browse/SI-7934 - private object DeprecationSuppressor extends DeprecationSuppressor - @deprecated("", "") - private class DeprecationSuppressor { - def initializeContainerModel(containerModel: ContainerModel, configModelRepo: ConfigModelRepo): Unit = { - containerModel.initialize(configModelRepo) - } - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala deleted file mode 100644 index 99cc8259ab3..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.config.{ConfigBuilder, ConfigInstance} -import com.yahoo.container.di.ConfigKeyT -import com.yahoo.container.di.config.{Subscriber, SubscriberFactory} -import com.yahoo.container.standalone.StandaloneSubscriberFactory._ -import com.yahoo.vespa.config.ConfigKey -import com.yahoo.vespa.model.VespaModel - -import scala.collection.JavaConverters._ - -/** - * @author tonytv - * @author gjoranv - */ -class StandaloneSubscriberFactory(root: VespaModel) extends SubscriberFactory { - class StandaloneSubscriber(configKeys: Set[ConfigKeyT]) extends Subscriber { - override def configChanged = - generation == 0 - - override def close() {} - - override def config = { - - def getConfig(key: ConfigKeyT) = { - val builderWithModelConfig = root.getConfig(newBuilderInstance(key), key.getConfigId) - - require(builderWithModelConfig != null, "Invalid config id " + key.getConfigId ) - (key.asInstanceOf[ConfigKey[ConfigInstance]], newConfigInstance(builderWithModelConfig)) - } - - (configKeys map getConfig).toMap.asJava - } - - override def waitNextGeneration() = { - generation += 1 - - if (generation != 0) { - while (!Thread.interrupted()) - Thread.sleep(10000) - } - - generation - } - - //if waitNextGeneration has not yet been called, -1 should be returned - var generation = -1L - } - - override def getSubscriber(configKeys: java.util.Set[_ <: ConfigKey[_]]) = - new StandaloneSubscriber(configKeys.asScala.toSet.asInstanceOf[Set[ConfigKeyT]]) - - def reloadActiveSubscribers(generation: Long) { - throw new RuntimeException("unsupported") - } -} - -object StandaloneSubscriberFactory { - - private def newBuilderInstance(key: ConfigKeyT) = - builderClass(key).getDeclaredConstructor().newInstance() - - private def builderClass(key: ConfigKeyT) = { - val nestedClasses = key.getConfigClass.getClasses - nestedClasses. - filter {_.getName.equals(key.getConfigClass.getName + "$Builder")}. - head. - asInstanceOf[Class[ConfigInstance.Builder]] - } - - private def newConfigInstance(builder: ConfigBuilder) = - configClass(builder).getConstructor(builder.getClass).newInstance(builder) - - private def configClass(builder: ConfigBuilder) = - builder.getClass.getEnclosingClass.asInstanceOf[Class[ConfigInstance]] - -} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java new file mode 100644 index 00000000000..1cd110d8106 --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigModelsPluginDir; +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServer; +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServers; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertThat; + +/** + * @author Ulf Lilleengen + * @author Tony Vaagenes + */ +public class CloudConfigInstallVariablesTest { + + @Test + public void test_configserver_parsing() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost.mydomain.com"); + assertThat(parsed.length, is(1)); + } + + @Test + public void port_can_be_configured() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost:123"); + int port = parsed[0].port.get(); + assertThat(port, is(123)); + } + + @Test + public void multiple_spaces_are_supported() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("test1 test2"); + assertThat(parsed.length, is(2)); + + List<String> hostNames = Arrays.stream(parsed).map(cs -> cs.hostName).collect(Collectors.toList()); + assertThat(hostNames, contains("test1", "test2")); + } + + @Test(expected = IllegalArgumentException.class) + public void missing_port_gives_exception() { + toConfigServer("myhost:"); + } + + @Test(expected = IllegalArgumentException.class) + public void non_numeric_port_gives_exception() { + toConfigServer("myhost:non-numeric"); + } + + @Test + public void string_arrays_are_split_on_spaces() { + String[] parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar "); + assertThat(parsed.length, is(2)); + } + + @Test + public void string_arrays_are_split_on_comma() { + String[] parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,"); + assertThat(parsed.length, is(2)); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java new file mode 100644 index 00000000000..a00ffd8b985 --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java @@ -0,0 +1,61 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +/** + * Creates a local application from vespa-services fragments. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneContainer { + public static String firstContainerId(AbstractConfigProducerRoot root) { + return root.getConfigProducer("container").get().getConfigId(); + } + + interface ThrowingFunction<T, U> { + U apply(T input) throws Exception; + } + + static <T> T withContainerModel(String servicesXml, ThrowingFunction<VespaModel, T> f) throws Exception { + return withTempDirectory(applicationPath -> { + writeServicesXml(applicationPath, servicesXml); + + LocalFileDb distributedFiles = new LocalFileDb(applicationPath); + VespaModel root; + Pair<VespaModel, com.yahoo.vespa.model.container.Container> rc = StandaloneContainerApplication.createContainerModel( + applicationPath, distributedFiles, applicationPath.resolve("preprocesedApp").toFile(), Networking.enable, + new ConfigModelRepo()); + root = rc.getFirst(); + return f.apply(root); + }); + } + + private static <T> T withTempDirectory(ThrowingFunction<Path, T> f) throws Exception { + Path directory = Files.createTempDirectory("application"); + try { + return f.apply(directory); + } finally { + IOUtils.recursiveDeleteDir(directory.toFile()); + } + } + + private static void writeServicesXml(Path applicationPath, String servicesXml) throws IOException { + Path path = applicationPath.resolve("services.xml"); + List<String> output = Arrays.asList("<?xml version=\"1.0\" encoding=\"utf-8\"?>", servicesXml); + Files.write(path, output, StandardCharsets.UTF_8); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java index 8d413ade0f0..71668b595a0 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.standalone; import com.google.inject.Module; @@ -25,7 +25,6 @@ import static org.junit.Assert.assertThat; /** * @author Einar M R Rosenvinge - * @since 5.22.0 */ public class StandaloneContainerActivatorTest { @@ -100,7 +99,7 @@ public class StandaloneContainerActivatorTest { private static Module newAppDirBinding(final Path applicationDir) { return binder -> binder.bind(Path.class) - .annotatedWith(StandaloneContainerApplication.applicationPathName()) + .annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME) .toInstance(applicationDir); } diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java new file mode 100644 index 00000000000..6d4abc84dbc --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java @@ -0,0 +1,74 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.vespa.model.AbstractService; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ + +public class StandaloneContainerTest { + private static final String PLAIN_XML = "<container version=\"1.0\" />"; + + @Test + public void container_is_allowed_root_element() throws Exception { + StandaloneContainer.withContainerModel(PLAIN_XML, root -> null); + } + + @Test + public void services_is_allowed_root_element() throws Exception { + String servicesXml = "<services>" + // + "<container version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(servicesXml, root -> null); + } + + @Test(expected = Exception.class) + public void multiple_container_elements_cannot_be_deployed() throws Exception { + String twoContainersXml = "<services>" + // + "<container id=\"container-1\" version=\"1.0\" />" + // + "<container id=\"container-2\" version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(twoContainersXml, root -> null); + } + + @Test + public void application_preprocessor_is_run() throws Exception { + String servicesXml = "<services xmlns:preprocess=\"properties\">" + // + "<preprocess:properties>" + // + "<container_id>container-1</container_id>" + // + "</preprocess:properties>" + // + "<container id=\"${container_id}\" version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(servicesXml, root -> { + assertTrue(root.getConfigProducer("container-1/standalone").isPresent()); + return null; + }); + } + + @Test + public void no_default_ports_are_enabled_when_using_http() throws Exception { + String xml = "<jdisc version=\"1.0\">" + // + "<http>" + // + "<server port=\"4000\" id=\"server1\" />" + // + "</http>" + // + "</jdisc>"; + + StandaloneContainer.withContainerModel(xml, root -> { + AbstractService container = (AbstractService) root.getConfigProducer("jdisc/standalone").get(); + System.out.println("portCnt: " + container.getPortCount()); + System.out.println("numPorts: " + container.getNumPortsAllocated()); + assertEquals(1, container.getNumPortsAllocated()); + return null; + }); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java new file mode 100644 index 00000000000..dd755e8e6dd --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.BundlesConfig; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.vespa.config.ConfigKey; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static com.yahoo.container.standalone.StandaloneContainer.withContainerModel; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.number.OrderingComparison.greaterThan; +import static org.junit.Assert.assertThat; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneSubscriberTest { + private static ConfigKey<ConfigInstance> bundlesKey = key("bundles"); + private static ConfigKey<ConfigInstance> componentsKey = key("components"); + + private static ConfigKey<ConfigInstance> key(String name) { + return new ConfigKey<>(name, "container", "container"); + } + + @Test + @Ignore + public void standalone_subscriber() throws Exception { + withContainerModel("<container version=\"1.0\"></container>", root -> { + Set<ConfigKey<ConfigInstance>> keys = new HashSet<>(); + keys.add(bundlesKey); + keys.add(componentsKey); + Subscriber subscriber = new StandaloneSubscriberFactory(root).getSubscriber(keys); + Map<ConfigKey<ConfigInstance>, ConfigInstance> config = subscriber.config(); + assertThat(config.size(), is(2)); + + BundlesConfig bundlesConfig = (BundlesConfig) config.get(bundlesKey); + ComponentsConfig componentsConfig = (ComponentsConfig) config.get(componentsKey); + + assertThat(bundlesConfig.bundle().size(), is(0)); + assertThat(componentsConfig.components().size(), greaterThan(10)); + return null; + }); + } +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala deleted file mode 100644 index d4baea43ba2..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.container.standalone.CloudConfigInstallVariables.{toConfigModelsPluginDir, toConfigServer, toConfigServers} -import org.hamcrest.CoreMatchers.is -import org.hamcrest.Matchers.arrayContaining -import org.junit.Assert.assertThat -import org.junit.Test - -/** - * @author Ulf Lilleengen - * @author Tony Vaagenes - */ -class CloudConfigInstallVariablesTest { - - @Test - def test_configserver_parsing { - val parsed = toConfigServers("myhost.mydomain.com") - assertThat(parsed.length, is(1)) - } - - @Test - def port_can_be_configured { - val parsed = toConfigServers("myhost:123") - val port: Int = parsed(0).port.get() - assertThat(port, is(123)) - } - - @Test - def multiple_spaces_are_supported { - val parsed = toConfigServers("test1 test2") - assertThat(parsed.size, is(2)) - - val hostNames = parsed.map(_.hostName) - assertThat(hostNames, arrayContaining("test1", "test2")) - } - - @Test(expected = classOf[IllegalArgumentException]) - def missing_port_gives_exception { - toConfigServer("myhost:") - } - - @Test(expected = classOf[IllegalArgumentException]) - def non_numeric_port_gives_exception { - toConfigServer("myhost:non-numeric") - } - - @Test - def string_arrays_are_split_on_spaces { - val parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar ") - assertThat(parsed.size, is(2)) - } - - @Test - def string_arrays_are_split_on_comma { - val parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,") - assertThat(parsed.size, is(2)) - } -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala deleted file mode 100644 index 33f9a2e8594..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.config.model.producer.AbstractConfigProducerRoot -import com.yahoo.config.model.test.MockRoot -import com.yahoo.container.Container -import com.yahoo.jdisc.test.TestDriver -import scala.xml.Node -import com.yahoo.vespa.model.VespaModel -import com.yahoo.io.IOUtils -import java.nio.file.{Files, Path} -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking - -/** - * Creates a local application from vespa-services fragments. - * - * @author tonytv - */ -object StandaloneContainer { - def firstContainerId(root: AbstractConfigProducerRoot): String = { - root.getConfigProducer("container").get().getConfigId - } - - def withStandaloneContainer[T](containerNode: Node) { - withTempDirectory { applicationDirectory => - System.setProperty(StandaloneContainerApplication.applicationLocationInstallVariable, applicationDirectory.toString) - createServicesXml(applicationDirectory, containerNode) - - val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication]) - driver.close() - Container.resetInstance() - } - } - - def withContainerModel[T](containerNode: Node)(f: VespaModel => T) { - withTempDirectory { applicationPath => - createServicesXml(applicationPath, containerNode) - - val distributedFiles = new LocalFileDb(applicationPath) - val (root, container) = StandaloneContainerApplication.createContainerModel( - applicationPath, - distributedFiles, - applicationPath.resolve("preprocesedApp").toFile, - networkingOption = Networking.enable) - f(root) - } - } - - private def withTempDirectory[T](f : Path => T) : T = { - val directory = Files.createTempDirectory("application") - try { - f(directory) - } finally { - IOUtils.recursiveDeleteDir(directory.toFile) - } - } - - private def createServicesXml(applicationPath : Path, - containerNode: Node) { - - scala.xml.XML.save(applicationPath.resolve("services.xml").toFile.getAbsolutePath, - containerNode, xmlDecl = true) - } -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala deleted file mode 100644 index 87bef2efd95..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - - -import com.yahoo.container.standalone.StandaloneContainerTest._ -import com.yahoo.vespa.model.AbstractService -import org.junit.Assert._ -import org.junit.Test - -import scala.util.Try - - -/** - * @author tonytv - * @author gjoranv - */ - -class StandaloneContainerTest { - @Test - def container_is_allowed_root_element() { - StandaloneContainer.withContainerModel(plainXml) { root => } - } - - @Test - def services_is_allowed_root_element() { - val servicesXml = - <services> - <container version="1.0" /> - </services> - - StandaloneContainer.withContainerModel(servicesXml) { root => } - } - - @Test - def multiple_container_elements_cannot_be_deployed() { - val twoContainersXml = - <services> - <container id="container-1" version="1.0" /> - <container id="container-2" version="1.0" /> - </services> - - assertTrue( - Try { - StandaloneContainer.withContainerModel(twoContainersXml) { root => } - }.isFailure) - } - - @Test - def application_preprocessor_is_run() { - val servicesXml = - <services xmlns:preprocess="properties"> - <preprocess:properties> - <container_id>container-1</container_id> - </preprocess:properties> - <container id="${container_id}" version="1.0" /> - </services> - StandaloneContainer.withContainerModel(servicesXml) { - root => - assertTrue(root.getConfigProducer("container-1/standalone").isPresent) - } - } - - @Test - def no_default_ports_are_enabled_when_using_http() { - val xml = - <jdisc version="1.0"> - <http> - <server port="4000" id="server1" /> - </http> - </jdisc> - - StandaloneContainer.withContainerModel(xml) { root => - val container = root.getConfigProducer("jdisc/standalone").get().asInstanceOf[AbstractService] - println("portCnt: " + container.getPortCount) - println("numPorts: " + container.getNumPortsAllocated) - assertEquals(1, container.getNumPortsAllocated) - } - } - -} - -object StandaloneContainerTest { - - val plainXml = <container version="1.0" /> -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala deleted file mode 100644 index ab6f486c748..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import org.junit.{Ignore, Test} -import org.junit.Assert.assertThat -import org.hamcrest.CoreMatchers.is -import org.hamcrest.number.OrderingComparison.greaterThan - -import StandaloneContainer.withContainerModel -import com.yahoo.vespa.config.ConfigKey -import com.yahoo.config.ConfigInstance -import com.yahoo.container.{ComponentsConfig, BundlesConfig, di} -import scala.collection.JavaConverters._ - -/** - * @author tonytv - */ -class StandaloneSubscriberTest { - val bundlesKey = key("bundles") - val componentsKey = key("components") - - def key(name: String) = new ConfigKey(name, "container", "container").asInstanceOf[ConfigKey[ConfigInstance]] - - def box(i: Int) = java.lang.Integer.valueOf(i) - - @Test - @Ignore - def standalone_subscriber() { - withContainerModel(<container version="1.0"> </container>) { root => - val subscriber = new StandaloneSubscriberFactory(root).getSubscriber(Set(bundlesKey, componentsKey).asJava) - val config = subscriber.config.asScala - assertThat(config.size, is(2)) - - val bundlesConfig = config(bundlesKey).asInstanceOf[BundlesConfig] - val componentsConfig = config(componentsKey).asInstanceOf[ComponentsConfig] - - assertThat(bundlesConfig.bundle().size(), is(0)) - assertThat(box(componentsConfig.components().size()), greaterThan(box(10))) - } - } -} diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index a271fbccf50..3b72585b076 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -14,6 +14,8 @@ #include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> using document::test::makeDocumentBucket; @@ -27,6 +29,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { void testRepliesAreDequeuedInFifoOrder(); void bucket_space_config_can_be_updated_live(); void unmapped_bucket_space_documentapi_request_returns_error_reply(); + void unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply(); static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60; @@ -51,6 +54,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder); CPPUNIT_TEST(bucket_space_config_can_be_updated_live); CPPUNIT_TEST(unmapped_bucket_space_documentapi_request_returns_error_reply); + CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply); CPPUNIT_TEST_SUITE_END(); }; @@ -267,13 +271,21 @@ struct CommunicationManagerFixture { } ~CommunicationManagerFixture(); - std::unique_ptr<documentapi::GetDocumentMessage> documentapi_message_for_space(const char* space) { - auto cmd = std::make_unique<documentapi::GetDocumentMessage>( - document::DocumentId(vespalib::make_string("id::%s::stuff", space))); + template <typename T> + std::unique_ptr<T> documentapi_message_for_space(const char *space) { + auto cmd = std::make_unique<T>(document::DocumentId(vespalib::make_string("id::%s::stuff", space))); // Bind reply handling to our own mock handler cmd->pushHandler(reply_handler); return cmd; } + + std::unique_ptr<documentapi::RemoveDocumentMessage> documentapi_remove_message_for_space(const char *space) { + return documentapi_message_for_space<documentapi::RemoveDocumentMessage>(space); + } + + std::unique_ptr<documentapi::GetDocumentMessage> documentapi_get_message_for_space(const char *space) { + return documentapi_message_for_space<documentapi::GetDocumentMessage>(space); + } }; CommunicationManagerFixture::~CommunicationManagerFixture() = default; @@ -298,8 +310,8 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { config.documenttype.emplace_back(doc_type("bar", "global")); f.comm_mgr->updateBucketSpacesConfig(config); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar")); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("foo")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("foo")); f.bottom_link->waitForMessages(2, MESSAGE_WAIT_TIME_SEC); auto cmd1 = f.bottom_link->getCommand(0); @@ -310,7 +322,7 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { config.documenttype[1] = doc_type("bar", "default"); f.comm_mgr->updateBucketSpacesConfig(config); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar")); f.bottom_link->waitForMessages(3, MESSAGE_WAIT_TIME_SEC); auto cmd3 = f.bottom_link->getCommand(2); @@ -328,7 +340,7 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns CPPUNIT_ASSERT_EQUAL(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("fluff")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("fluff")); CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); auto& reply = *f.reply_handler.replies[0]; CPPUNIT_ASSERT(reply.hasErrors()); @@ -337,4 +349,23 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } +// Legacy DocumentAPI routing protocols will send Gets to _all_ clusters even +// if they do not contain a particular document type. By sending an empty reply +// we signal a mergeable "not found" to the sender rather than a non-mergeable +// fatal error. +void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply() { + CommunicationManagerFixture f; + + BucketspacesConfigBuilder config; + config.documenttype.emplace_back(doc_type("foo", "default")); + f.comm_mgr->updateBucketSpacesConfig(config); + + f.comm_mgr->handleMessage(f.documentapi_get_message_for_space("fluff")); + CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); + auto& reply = *f.reply_handler.replies[0]; + CPPUNIT_ASSERT(!reply.hasErrors()); + auto& get_reply = dynamic_cast<documentapi::GetDocumentReply&>(reply); + CPPUNIT_ASSERT(!get_reply.hasDocument()); +} + } // storage diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 8b17e851868..b9d4f24072d 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -19,9 +19,14 @@ #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/config/common/exceptions.h> +#include <optional> +#include <thread> +#include <chrono> using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using documentapi::Priority; +using namespace std::chrono_literals; namespace storage { namespace { @@ -78,8 +83,9 @@ public: std::vector<document::Document::SP >& docs, std::vector<document::DocumentId>& docIds, api::ReturnCode::Result returnCode = api::ReturnCode::OK, - documentapi::Priority::Value priority = documentapi::Priority::PRI_NORMAL_4); + std::optional<Priority::Value> priority = documentapi::Priority::PRI_NORMAL_4); uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); + void finishAndWaitForVisitorSessionCompletion(uint32_t sessionIndex); void testNormalUsage(); void testResending(); @@ -185,8 +191,7 @@ VisitorManagerTest::initializeTest() for (uint32_t i=0; i<10; ++i) { document::BucketId bid(16, i); - std::shared_ptr<api::CreateBucketCommand> cmd( - new api::CreateBucketCommand(makeDocumentBucket(bid))); + auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); cmd->setAddress(address); cmd->setSourceIndex(0); _top->sendDown(cmd); @@ -202,8 +207,7 @@ VisitorManagerTest::initializeTest() for (uint32_t i=0; i<docCount; ++i) { document::BucketId bid(16, i); - std::shared_ptr<api::PutCommand> cmd( - new api::PutCommand(makeDocumentBucket(bid), _documents[i], i+1)); + auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), _documents[i], i+1); cmd->setAddress(address); _top->sendDown(cmd); _top->waitForMessages(1, 60); @@ -226,45 +230,40 @@ VisitorManagerTest::addSomeRemoves(bool removeAll) for (uint32_t i=0; i<docCount; i += (removeAll ? 1 : 4)) { // Add it to the database document::BucketId bid(16, i % 10); - std::shared_ptr<api::RemoveCommand> cmd( - new api::RemoveCommand( - makeDocumentBucket(bid), _documents[i]->getId(), clock.getTimeInMicros().getTime() + docCount + i + 1)); + auto cmd = std::make_shared<api::RemoveCommand>( + makeDocumentBucket(bid), _documents[i]->getId(), clock.getTimeInMicros().getTime() + docCount + i + 1); cmd->setAddress(address); _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); - std::shared_ptr<api::RemoveReply> reply( - std::dynamic_pointer_cast<api::RemoveReply>( - replies[0])); + CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + auto reply = std::dynamic_pointer_cast<api::RemoveReply>(replies[0]); CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), - reply->getResult()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); } } void VisitorManagerTest::tearDown() { - if (_top.get() != 0) { + if (_top) { + assert(_top->getNumReplies() == 0); _top->close(); _top->flush(); - _top.reset(0); + _top.reset(); } - _node.reset(0); - _messageSessionFactory.reset(0); - _manager = 0; + _node.reset(); + _messageSessionFactory.reset(); + _manager = nullptr; } TestVisitorMessageSession& VisitorManagerTest::getSession(uint32_t n) { // Wait until we have started the visitor - const std::vector<TestVisitorMessageSession*>& sessions( - _messageSessionFactory->_visitorSessions); + const std::vector<TestVisitorMessageSession*>& sessions(_messageSessionFactory->_visitorSessions); framework::defaultimplementation::RealClock clock; - framework::MilliSecTime endTime( - clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); + framework::MilliSecTime endTime(clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); while (true) { { vespalib::LockGuard lock(_messageSessionFactory->_accessLock); @@ -276,7 +275,7 @@ VisitorManagerTest::getSession(uint32_t n) throw vespalib::IllegalStateException( "Timed out waiting for visitor session", VESPA_STRLOC); } - FastOS_Thread::Sleep(10); + std::this_thread::sleep_for(10ms); } throw std::logic_error("unreachable"); } @@ -288,7 +287,7 @@ VisitorManagerTest::getMessagesAndReply( std::vector<document::Document::SP >& docs, std::vector<document::DocumentId>& docIds, api::ReturnCode::Result result, - documentapi::Priority::Value priority) + std::optional<Priority::Value> priority) { for (int i = 0; i < expectedCount; i++) { session.waitForMessages(i + 1); @@ -296,8 +295,10 @@ VisitorManagerTest::getMessagesAndReply( { vespalib::MonitorGuard guard(session.getMonitor()); - CPPUNIT_ASSERT_EQUAL(priority, - session.sentMessages[i]->getPriority()); + if (priority) { + CPPUNIT_ASSERT_EQUAL(*priority, + session.sentMessages[i]->getPriority()); + } switch (session.sentMessages[i]->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: @@ -340,8 +341,7 @@ VisitorManagerTest::verifyCreateVisitorReply( CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); CPPUNIT_ASSERT(reply.get()); CPPUNIT_ASSERT_EQUAL(expectedResult, reply->getResult().getResult()); @@ -410,8 +410,7 @@ VisitorManagerTest::testNormalUsage() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setControlDestination("foo/bar"); @@ -436,8 +435,7 @@ VisitorManagerTest::testResending() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setControlDestination("foo/bar"); @@ -486,8 +484,7 @@ VisitorManagerTest::testVisitEmptyBucket() initializeTest(); addSomeRemoves(true); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); @@ -502,8 +499,7 @@ VisitorManagerTest::testMultiBucketVisit() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); } @@ -527,8 +523,7 @@ VisitorManagerTest::testNoBuckets() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->setAddress(address); _top->sendDown(cmd); @@ -536,15 +531,12 @@ VisitorManagerTest::testNoBuckets() // Should get one reply; a CreateVisitorReply with error since no // buckets where specified in the CreateVisitorCommand _top->waitForMessages(1, 60); - const msg_ptr_vector replies = _top->getRepliesOnce(); + const msg_ptr_vector replies = _top->getRepliesOnce(); CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>( - replies[0])); + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(replies[0]); // Verify that cast went ok => it was a CreateVisitorReply message CPPUNIT_ASSERT(reply.get()); - api::ReturnCode ret(api::ReturnCode::ILLEGAL_PARAMETERS, - "No buckets specified"); + api::ReturnCode ret(api::ReturnCode::ILLEGAL_PARAMETERS, "No buckets specified"); CPPUNIT_ASSERT_EQUAL(ret, reply->getResult()); } @@ -553,8 +545,7 @@ void VisitorManagerTest::testVisitPutsAndRemoves() initializeTest(); addSomeRemoves(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->setAddress(address); cmd->setVisitRemoves(); for (uint32_t i=0; i<10; ++i) { @@ -581,9 +572,7 @@ void VisitorManagerTest::testVisitWithTimeframeAndSelection() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", - "testdoctype1.headerval < 2")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval < 2"); cmd->setFromTime(3); cmd->setToTime(8); for (uint32_t i=0; i<10; ++i) { @@ -613,9 +602,8 @@ void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() { initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", - "DocType(testdoctype1---///---) XXX BAD Field(headerval) < 2")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", + "DocType(testdoctype1---///---) XXX BAD Field(headerval) < 2"); cmd->setFromTime(3); cmd->setToTime(8); for (uint32_t i=0; i<10; ++i) { @@ -628,11 +616,9 @@ void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() const msg_ptr_vector replies = _top->getRepliesOnce(); CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); - api::StorageReply* reply = dynamic_cast<api::StorageReply*>( - replies.front().get()); + auto* reply = dynamic_cast<api::StorageReply*>(replies.front().get()); CPPUNIT_ASSERT(reply); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ILLEGAL_PARAMETERS, - reply->getResult().getResult()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ILLEGAL_PARAMETERS, reply->getResult().getResult()); } void @@ -641,8 +627,7 @@ VisitorManagerTest::testVisitorCallbacks() initializeTest(); std::ostringstream replydata; api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "TestVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "TestVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->addBucketToBeVisited(document::BucketId(16, 5)); cmd->setAddress(address); @@ -659,8 +644,8 @@ VisitorManagerTest::testVisitorCallbacks() CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_MAPVISITOR, session.sentMessages[i]->getType()); - documentapi::MapVisitorMessage* mapvisitormsg( - static_cast<documentapi::MapVisitorMessage*>(session.sentMessages[i].get())); + auto* mapvisitormsg = dynamic_cast<documentapi::MapVisitorMessage*>(session.sentMessages[i].get()); + CPPUNIT_ASSERT(mapvisitormsg != nullptr); replydata << mapvisitormsg->getData().get("msg"); @@ -690,8 +675,7 @@ VisitorManagerTest::testVisitorCleanup() for (uint32_t i=0; i<10; ++i) { std::ostringstream ost; ost << "testvis" << i; - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "InvalidVisitor", ost.str(), "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "InvalidVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -703,28 +687,27 @@ VisitorManagerTest::testVisitorCleanup() for (uint32_t i=0; i<10; ++i) { std::ostringstream ost; ost << "testvis" << (i + 10); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); _top->sendDown(cmd); } - - // Should get 14 immediate replies - 10 failures and 4 busy + // Should get 16 immediate replies - 10 failures and 6 busy { - _top->waitForMessages(14, 60); + const int expected_total = 16; + _top->waitForMessages(expected_total, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL(size_t(expected_total), replies.size()); int failures = 0; int busy = 0; - for (uint32_t i=0; i< 14; ++i) { + for (uint32_t i=0; i< expected_total; ++i) { std::shared_ptr<api::StorageMessage> msg(replies[i]); CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); CPPUNIT_ASSERT(reply.get()); if (i < 10) { @@ -741,9 +724,11 @@ VisitorManagerTest::testVisitorCleanup() } CPPUNIT_ASSERT_EQUAL(10, failures); - CPPUNIT_ASSERT_EQUAL(4, busy); + CPPUNIT_ASSERT_EQUAL(expected_total - 10, busy); } + // 4 pending + // Finish a visitor std::vector<document::Document::SP > docs; std::vector<document::DocumentId> docIds; @@ -753,22 +738,25 @@ VisitorManagerTest::testVisitorCleanup() // Should get a reply for the visitor. verifyCreateVisitorReply(api::ReturnCode::OK); + // 3 pending + // Fail a visitor getMessagesAndReply(1, getSession(1), docs, docIds, api::ReturnCode::INTERNAL_FAILURE); // Should get a reply for the visitor. verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE); - while (_manager->getActiveVisitorCount() > 2) { - FastOS_Thread::Sleep(10); + // Wait until there are 2 pending. Visitor threads might not have completed + // cleanup of existing visitors yet. + while (_manager->getActiveVisitorCount() != 2) { + std::this_thread::sleep_for(10ms); } // Start a bunch of more visitors for (uint32_t i=0; i<10; ++i) { std::ostringstream ost; ost << "testvis" << (i + 24); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -778,17 +766,21 @@ VisitorManagerTest::testVisitorCleanup() // Should now get 8 busy. _top->waitForMessages(8, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); - CPPUNIT_ASSERT_EQUAL(8, (int)replies.size()); + CPPUNIT_ASSERT_EQUAL(size_t(8), replies.size()); for (uint32_t i=0; i< replies.size(); ++i) { std::shared_ptr<api::StorageMessage> msg(replies[i]); CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); CPPUNIT_ASSERT(reply.get()); CPPUNIT_ASSERT_EQUAL(api::ReturnCode::BUSY, reply->getResult().getResult()); } + + for (uint32_t i = 0; i < 4; ++i) { + getMessagesAndReply(1, getSession(i + 2), docs, docIds); + verifyCreateVisitorReply(api::ReturnCode::OK); + } } void @@ -798,18 +790,13 @@ VisitorManagerTest::testAbortOnFailedVisitorInfo() api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); { - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); _top->sendDown(cmd); } - uint32_t visitorRepliesReceived = 0; - uint32_t oki = 0; - uint32_t failed = 0; - std::vector<document::Document::SP > docs; std::vector<document::DocumentId> docIds; @@ -823,37 +810,13 @@ VisitorManagerTest::testAbortOnFailedVisitorInfo() mbus::Reply::UP reply = cmd->createReply(); - CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_VISITORINFO, session.sentMessages[1]->getType()); + CPPUNIT_ASSERT_EQUAL(uint32_t(documentapi::DocumentProtocol::MESSAGE_VISITORINFO), session.sentMessages[1]->getType()); reply->swapState(*session.sentMessages[1]); reply->setMessage(mbus::Message::UP(session.sentMessages[1].release())); reply->addError(mbus::Error(api::ReturnCode::NOT_CONNECTED, "Me no ready")); session.reply(std::move(reply)); } - - _top->waitForMessages(1, 60); - const msg_ptr_vector replies = _top->getRepliesOnce(); - for (uint32_t i=0; i< replies.size(); ++i) { - std::shared_ptr<api::StorageMessage> msg(replies[i]); - if (msg->getType() == api::MessageType::VISITOR_CREATE_REPLY) - { - ++visitorRepliesReceived; - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); - CPPUNIT_ASSERT(reply.get()); - if (reply->getResult().success()) { - ++oki; - std::cerr << "\n" << reply->toString(true) << "\n"; - } else { - ++failed; - } - } - } - - std::ostringstream errmsg; - errmsg << "oki " << oki << ", failed " << failed; - - CPPUNIT_ASSERT_EQUAL_MSG(errmsg.str(), 0u, oki); - CPPUNIT_ASSERT_EQUAL_MSG(errmsg.str(), 1u, failed); + verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED); } void @@ -863,10 +826,8 @@ VisitorManagerTest::testAbortOnFieldPathError() api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); // Use bogus field path to force error to happen - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", - "testvis", - "testdoctype1.headerval{bogus} == 1234")); + auto cmd = std::make_shared<api::CreateVisitorCommand>( + makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval{bogus} == 1234"); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -885,8 +846,7 @@ VisitorManagerTest::testVisitorQueueTimeout() { vespalib::MonitorGuard guard(_manager->getThread(0).getQueueMonitor()); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(1); @@ -897,18 +857,13 @@ VisitorManagerTest::testVisitorQueueTimeout() } // Don't answer any messages. Make sure we timeout anyways. - uint32_t visitorRepliesReceived = 0; - _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); std::shared_ptr<api::StorageMessage> msg(replies[0]); CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - ++visitorRepliesReceived; - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::BUSY, - "Visitor timed out in visitor queue"), + auto reply = std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::BUSY, "Visitor timed out in visitor queue"), reply->getResult()); } @@ -918,8 +873,7 @@ VisitorManagerTest::testVisitorProcessingTimeout() initializeTest(); api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", "testvis", "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(0); @@ -932,19 +886,7 @@ VisitorManagerTest::testVisitorProcessingTimeout() _node->getClock().addSecondsToTime(1000); - // Don't answer any messages. Make sure we timeout anyways. - uint32_t visitorRepliesReceived = 0; - - _top->waitForMessages(1, 60); - const msg_ptr_vector replies = _top->getRepliesOnce(); - std::shared_ptr<api::StorageMessage> msg(replies[0]); - - CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); - ++visitorRepliesReceived; - std::shared_ptr<api::CreateVisitorReply> reply( - std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, - reply->getResult().getResult()); + verifyCreateVisitorReply(api::ReturnCode::ABORTED); } namespace { @@ -955,8 +897,7 @@ namespace { std::ostringstream ost; ost << "testvis" << ++nextVisitor; api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); - std::shared_ptr<api::CreateVisitorCommand> cmd( - new api::CreateVisitorCommand(makeBucketSpace(), "DumpVisitor", ost.str(), "")); + auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->setAddress(address); cmd->setQueueTimeout(timeout); @@ -1002,14 +943,25 @@ VisitorManagerTest::testPrioritizedVisitorQueing() // Finish the first visitor std::vector<document::Document::SP > docs; std::vector<document::DocumentId> docIds; - getMessagesAndReply(1, getSession(0), docs, docIds, api::ReturnCode::OK, - documentapi::Priority::PRI_HIGHEST); + getMessagesAndReply(1, getSession(0), docs, docIds, api::ReturnCode::OK, Priority::PRI_HIGHEST); verifyCreateVisitorReply(api::ReturnCode::OK); // We should now start the highest priority visitor. - getMessagesAndReply(1, getSession(4), docs, docIds, api::ReturnCode::OK, - documentapi::Priority::PRI_VERY_HIGH); + getMessagesAndReply(1, getSession(4), docs, docIds, api::ReturnCode::OK, Priority::PRI_VERY_HIGH); CPPUNIT_ASSERT_EQUAL(ids[9], verifyCreateVisitorReply(api::ReturnCode::OK)); + + // 3 pending, 3 in queue. Clean them up + std::vector<uint32_t> pending_sessions = {1, 2, 3, 5, 6, 7}; + for (auto session : pending_sessions) { + finishAndWaitForVisitorSessionCompletion(session); + } +} + +void VisitorManagerTest::finishAndWaitForVisitorSessionCompletion(uint32_t sessionIndex) { + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + getMessagesAndReply(1, getSession(sessionIndex), docs, docIds, api::ReturnCode::OK, std::optional<Priority::Value>()); + verifyCreateVisitorReply(api::ReturnCode::OK); } void @@ -1135,6 +1087,9 @@ VisitorManagerTest::testVisitorQueingZeroQueueSize() { sendCreateVisitor(1000, *_top, 100 - i); verifyCreateVisitorReply(api::ReturnCode::BUSY); } + for (uint32_t session = 0; session < 4; ++session) { + finishAndWaitForVisitorSessionCompletion(session); + } } void @@ -1148,8 +1103,10 @@ VisitorManagerTest::testStatusPage() { sendCreateVisitor(1000000, *_top, 1); sendCreateVisitor(1000000, *_top, 128); - TestVisitorMessageSession& session = getSession(0); - session.waitForMessages(1); + { + TestVisitorMessageSession& session = getSession(0); + session.waitForMessages(1); + } std::ostringstream ss; static_cast<framework::HtmlStatusReporter&>(*_manager).reportHtmlStatus(ss, path); @@ -1162,6 +1119,10 @@ VisitorManagerTest::testStatusPage() { CPPUNIT_ASSERT(str.find("Visitor thread 0") != std::string::npos); CPPUNIT_ASSERT(str.find("Disconnected visitor timeout") != std::string::npos); // verbose per thread CPPUNIT_ASSERT(str.find("Message #1 <b>putdocumentmessage</b>") != std::string::npos); // 1 active + + for (uint32_t session = 0; session < 2 ; ++session){ + finishAndWaitForVisitorSessionCompletion(session); + } } } diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index a08445ca3d2..471ce7e2b27 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -236,6 +236,7 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request if (result == api::ReturnCode::Result::ENCODE_ERROR) { // Handle failure to encode bucket space due to use of old storage api // protocol. Pretend that request succeeded with no buckets returned. + // TODO remove this workaround for Vespa 7 LOG(debug, "Got ENCODE_ERROR, pretending success with no buckets"); } else if (!result.success()) { framework::MilliSecTime resendTime(_clock); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index e87eabd19df..06dfc073f61 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -569,7 +569,7 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck } // Follow onto next to move queue or fail } - //@fallthrough@ + [[fallthrough]]; case api::MessageType::SPLITBUCKET_ID: // Move to correct queue if op == MOVE // Fail with bucket not found if op is JOIN @@ -640,7 +640,7 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck break; case GetIterCommand::ID: bucket = static_cast<GetIterCommand&>(msg).getBucket(); - //@fallthrough@ + [[fallthrough]]; case RepairBucketCommand::ID: if (bucket.getBucketId().getRawId() == 0) { bucket = static_cast<RepairBucketCommand&>(msg).getBucket(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 63c957207a6..45ac5ded47f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -190,8 +190,14 @@ public: std::string dumpQueue() const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; + static uint64_t dispersed_bucket_bits(const document::Bucket& bucket) noexcept { + // Disperse bucket bits by multiplying with the 64-bit FNV-1 prime. + // This avoids an inherent affinity between the LSB of a bucket's bits + // and the stripe an operation ends up on. + return bucket.getBucketId().getRawId() * 1099511628211ULL; + } Stripe & stripe(const document::Bucket & bucket) { - return _stripes[bucket.getBucketId().getRawId()%_stripes.size()]; + return _stripes[dispersed_bucket_bits(bucket) % _stripes.size()]; } std::vector<Stripe> & getStripes() { return _stripes; } private: diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 22193d7e246..94a151bcdc1 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -19,6 +19,7 @@ #include <vespa/log/bufferedlogger.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> LOG_SETUP(".communication.manager"); @@ -258,8 +259,17 @@ void CommunicationManager::fail_with_unresolvable_bucket_space( { LOG(debug, "Could not map DocumentAPI message to internal bucket: %s", error_message.c_str()); MBUS_TRACE(msg->getTrace(), 6, "Communication manager: Failing message as its document type has no known bucket space mapping"); - std::unique_ptr<mbus::Reply> reply(new mbus::EmptyReply()); - reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); + std::unique_ptr<mbus::Reply> reply; + if (msg->getType() == documentapi::DocumentProtocol::MESSAGE_GETDOCUMENT) { + // HACK: to avoid breaking legacy routing of GetDocumentMessages to _all_ clusters + // regardless of them having a document type or not, we remap missing bucket spaces + // to explicit Not Found replies (empty document GetDocumentReply). + // TODO remove this workaround for Vespa 7 + reply = std::make_unique<documentapi::GetDocumentReply>(std::shared_ptr<document::Document>()); + } else { + reply = std::make_unique<mbus::EmptyReply>(); + reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); + } msg->swapState(*reply); _metrics.bucketSpaceMappingFailures.inc(); _messageBusSession->reply(std::move(reply)); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index ad98d64b173..6bb2ca31ec1 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -206,10 +206,8 @@ StorageNode::initialize() _chain.reset(createChain().release()); - if (_component->enableMultipleBucketSpaces()) { - assert(_communicationManager != nullptr); - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + assert(_communicationManager != nullptr); + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); // Start the metric manager, such that it starts generating snapshots // and the like. Note that at this time, all metrics should hopefully @@ -359,9 +357,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) if (_newBucketSpacesConfig) { _bucketSpacesConfig = std::move(_newBucketSpacesConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - if (_component->enableMultipleBucketSpaces()) { - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index f1e96cc1631..a1be0def20b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -60,8 +60,7 @@ ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const } void -ProtocolSerialization5_0::putBucketInfo( - const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const +ProtocolSerialization5_0::putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const { buf.putInt(info.getChecksum()); buf.putInt(info.getDocumentCount()); @@ -71,8 +70,7 @@ ProtocolSerialization5_0::putBucketInfo( } void -ProtocolSerialization5_0::onEncodeReply( - GBBuf& buf, const api::StorageReply& msg) const +ProtocolSerialization5_0::onEncodeReply(GBBuf& buf, const api::StorageReply& msg) const { SH::putReturnCode(msg.getResult(), buf); buf.putLong(msg.getMsgId()); @@ -80,8 +78,7 @@ ProtocolSerialization5_0::onEncodeReply( } void -ProtocolSerialization5_0::onDecodeReply(BBuf& buf, - api::StorageReply& msg) const +ProtocolSerialization5_0::onDecodeReply(BBuf& buf, api::StorageReply& msg) const { msg.setResult(SH::getReturnCode(buf)); msg.forceMsgId(SH::getLong(buf)); @@ -89,8 +86,7 @@ ProtocolSerialization5_0::onDecodeReply(BBuf& buf, } void -ProtocolSerialization5_0::onEncodeCommand( - GBBuf& buf, const api::StorageCommand& msg) const +ProtocolSerialization5_0::onEncodeCommand(GBBuf& buf, const api::StorageCommand& msg) const { buf.putLong(msg.getMsgId()); buf.putByte(msg.getPriority()); @@ -99,8 +95,7 @@ ProtocolSerialization5_0::onEncodeCommand( } void -ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, - api::StorageCommand& msg) const +ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const { msg.forceMsgId(SH::getLong(buf)); uint8_t priority = SH::getByte(buf); @@ -118,15 +113,13 @@ ProtocolSerialization5_0::ProtocolSerialization5_0( { } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::PutReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutReply& msg) const { buf.putBoolean(msg.wasFound()); onEncodeBucketInfoReply(buf, msg); } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::PutCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutCommand& msg) const { SH::putDocument(msg.getDocument().get(), buf); putBucket(msg.getBucket(), buf); @@ -141,24 +134,22 @@ ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); document::Bucket bucket = getBucket(buf); api::Timestamp ts(SH::getLong(buf)); - api::PutCommand::UP msg(new api::PutCommand(bucket, doc, ts)); + auto msg = std::make_unique<api::PutCommand>(bucket, doc, ts); msg->setUpdateTimestamp(SH::getLong(buf)); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } api::StorageReply::UP ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const { bool wasFound = SH::getBoolean(buf); - api::PutReply::UP msg(new api::PutReply( - static_cast<const api::PutCommand&>(cmd), wasFound)); + auto msg = std::make_unique<api::PutReply>(static_cast<const api::PutCommand&>(cmd), wasFound); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::UpdateReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateReply& msg) const { buf.putLong(msg.getOldTimestamp()); onEncodeBucketInfoReply(buf, msg); @@ -168,14 +159,12 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const { api::Timestamp oldTimestamp(SH::getLong(buf)); - api::UpdateReply::UP msg(new api::UpdateReply( - static_cast<const api::UpdateCommand&>(cmd), oldTimestamp)); + auto msg = std::make_unique<api::UpdateReply>(static_cast<const api::UpdateCommand&>(cmd), oldTimestamp); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::GetReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetReply& msg) const { SH::putDocument(msg.getDocument().get(), buf); buf.putLong(msg.getLastModifiedTimestamp()); @@ -188,28 +177,17 @@ ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const try { document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); api::Timestamp lastModified(SH::getLong(buf)); - api::GetReply::UP msg( - new api::GetReply( - static_cast<const api::GetCommand&>(cmd), - doc, - lastModified)); + auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), doc,lastModified); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } catch (std::exception& e) { - api::GetReply::UP msg( - new api::GetReply( - static_cast<const api::GetCommand&>(cmd), - document::Document::SP(), - 0)); - msg->setResult(api::ReturnCode( - api::ReturnCode::UNPARSEABLE, - e.what())); - return api::StorageReply::UP(msg.release()); + auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), document::Document::SP(),0); + msg->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); + return msg; } } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RemoveReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RemoveReply& msg) const { buf.putLong(msg.getOldTimestamp()); onEncodeBucketInfoReply(buf, msg); @@ -219,14 +197,12 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const { api::Timestamp oldTimestamp(SH::getLong(buf)); - api::RemoveReply::UP msg(new api::RemoveReply( - static_cast<const api::RemoveCommand&>(cmd), oldTimestamp)); + auto msg = std::make_unique<api::RemoveReply>(static_cast<const api::RemoveCommand&>(cmd), oldTimestamp); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::UpdateCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const { document::DocumentUpdate* update = msg.getUpdate().get(); if (update) { @@ -253,24 +229,19 @@ ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const if (size != 0) { document::ByteBuffer bbuf(buf.getBufferAtPos(), size); buf.incPos(size); - update.reset(new document::DocumentUpdate(getTypeRepo(), bbuf, - document::DocumentUpdate:: - SerializeVersion:: - SERIALIZE_HEAD)); + update = document::DocumentUpdate::createHEAD(getTypeRepo(), bbuf); } document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); - api::UpdateCommand::UP msg( - new api::UpdateCommand(bucket, update, timestamp)); + api::UpdateCommand::UP msg = std::make_unique<api::UpdateCommand>(bucket, update, timestamp); msg->setOldTimestamp(SH::getLong(buf)); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RevertReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RevertReply& msg) const { onEncodeBucketInfoReply(buf, msg); } @@ -278,31 +249,26 @@ void ProtocolSerialization5_0::onEncode( api::StorageReply::UP ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const { - api::RevertReply::UP msg(new api::RevertReply( - static_cast<const api::RevertCommand&>(cmd))); + auto msg = std::make_unique<api::RevertReply>(static_cast<const api::RevertCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::CreateBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const { onEncodeBucketInfoReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const { - api::CreateBucketReply::UP msg(new api::CreateBucketReply( - static_cast<const api::CreateBucketCommand&>(cmd))); + auto msg = std::make_unique<api::CreateBucketReply>(static_cast<const api::CreateBucketCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::DeleteBucketCommand& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const { putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); @@ -313,32 +279,28 @@ api::StorageCommand::UP ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bucket)); + auto msg = std::make_unique<api::DeleteBucketCommand>(bucket); onDecodeBucketInfoCommand(buf, *msg); if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) { msg->setBucketInfo(getBucketInfo(buf)); } - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::DeleteBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const { onEncodeBucketInfoReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const { - api::DeleteBucketReply::UP msg(new api::DeleteBucketReply( - static_cast<const api::DeleteBucketCommand&>(cmd))); + auto msg = std::make_unique<api::DeleteBucketReply>(static_cast<const api::DeleteBucketCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::MergeBucketCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { ProtocolSerialization4_2::onEncode(buf, msg); @@ -353,8 +315,7 @@ void ProtocolSerialization5_0::onEncode( api::StorageCommand::UP ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const { - api::StorageCommand::UP cmd - = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); + api::StorageCommand::UP cmd = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); uint32_t clusterStateVersion = SH::getInt(buf); uint16_t chainSize = SH::getShort(buf); std::vector<uint16_t> chain; @@ -369,24 +330,20 @@ ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const return cmd; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::MergeBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const { onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const { - api::MergeBucketReply::UP msg(new api::MergeBucketReply( - static_cast<const api::MergeBucketCommand&>(cmd))); + auto msg = std::make_unique<api::MergeBucketReply>(static_cast<const api::MergeBucketCommand&>(cmd)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::GetBucketDiffReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const { const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); @@ -397,11 +354,9 @@ void ProtocolSerialization5_0::onEncode( } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - api::GetBucketDiffReply::UP msg(new api::GetBucketDiffReply( - static_cast<const api::GetBucketDiffCommand&>(cmd))); + auto msg = std::make_unique<api::GetBucketDiffReply>(static_cast<const api::GetBucketDiffCommand&>(cmd)); std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -413,24 +368,20 @@ ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, onDecodeDiffEntry(buf, entries[i]); } onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::ApplyBucketDiffReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const { - const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( - msg.getDiff()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { onEncodeDiffEntry(buf, entries[i]._entry); buf.putString(entries[i]._docName); buf.putInt(entries[i]._headerBlob.size()); - buf.putBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); buf.putInt(entries[i]._bodyBlob.size()); - buf.putBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onEncodeBucketInfoReply(buf, msg); } @@ -439,8 +390,7 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - api::ApplyBucketDiffReply::UP msg(new api::ApplyBucketDiffReply( - static_cast<const api::ApplyBucketDiffCommand&>(cmd))); + auto msg = std::make_unique<api::ApplyBucketDiffReply>(static_cast<const api::ApplyBucketDiffCommand&>(cmd)); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -456,25 +406,21 @@ ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd, buf.incPos(headerSize); } entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); uint32_t bodySize = SH::getInt(buf); if (bodySize > buf.getRemaining()) { buf.incPos(bodySize); } entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::SplitBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const { - const std::vector<api::SplitBucketReply::Entry>& entries( - msg.getSplitInfo()); + const std::vector<api::SplitBucketReply::Entry>& entries(msg.getSplitInfo()); buf.putInt(entries.size()); for (std::vector<api::SplitBucketReply::Entry>::const_iterator it = entries.begin(); it != entries.end(); ++it) @@ -486,11 +432,9 @@ void ProtocolSerialization5_0::onEncode( } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const { - api::SplitBucketReply::UP msg(new api::SplitBucketReply( - static_cast<const api::SplitBucketCommand&>(cmd))); + auto msg = std::make_unique<api::SplitBucketReply>(static_cast<const api::SplitBucketCommand&>(cmd)); std::vector<api::SplitBucketReply::Entry>& entries(msg->getSplitInfo()); uint32_t targetCount = SH::getInt(buf); if (targetCount > buf.getRemaining()) { @@ -505,12 +449,11 @@ ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, it->second = getBucketInfo(buf); } onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::JoinBucketsCommand& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putInt(msg.getSourceBuckets().size()); @@ -525,7 +468,7 @@ api::StorageCommand::UP ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket)); + auto msg = std::make_unique<api::JoinBucketsCommand>(bucket); uint32_t size = SH::getInt(buf); if (size > buf.getRemaining()) { // Trigger out of bounds exception rather than out of memory error @@ -537,55 +480,48 @@ ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const } msg->setMinJoinBits(SH::getByte(buf)); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::JoinBucketsReply& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const { putBucketInfo(msg.getBucketInfo(), buf); onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const { - api::JoinBucketsReply::UP msg(new api::JoinBucketsReply( - static_cast<const api::JoinBucketsCommand&>(cmd))); + auto msg = std::make_unique<api::JoinBucketsReply>(static_cast<const api::JoinBucketsCommand&>(cmd)); msg->setBucketInfo(getBucketInfo(buf)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncodeBucketInfoReply( - GBBuf& buf, const api::BucketInfoReply& msg) const +ProtocolSerialization5_0::onEncodeBucketInfoReply(GBBuf& buf, const api::BucketInfoReply& msg) const { onEncodeBucketReply(buf, msg); putBucketInfo(msg.getBucketInfo(), buf); } void -ProtocolSerialization5_0::onDecodeBucketInfoReply( - BBuf& buf, api::BucketInfoReply& msg) const +ProtocolSerialization5_0::onDecodeBucketInfoReply(BBuf& buf, api::BucketInfoReply& msg) const { onDecodeBucketReply(buf, msg); msg.setBucketInfo(getBucketInfo(buf)); } void -ProtocolSerialization5_0::onEncodeBucketReply( - GBBuf& buf, const api::BucketReply& msg) const +ProtocolSerialization5_0::onEncodeBucketReply(GBBuf& buf, const api::BucketReply& msg) const { onEncodeReply(buf, msg); buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0); } void -ProtocolSerialization5_0::onDecodeBucketReply( - BBuf& buf, api::BucketReply& msg) const +ProtocolSerialization5_0::onDecodeBucketReply(BBuf& buf, api::BucketReply& msg) const { onDecodeReply(buf, msg); document::BucketId bucket(SH::getLong(buf)); @@ -608,11 +544,9 @@ ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& ms } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const { - api::CreateVisitorReply::UP msg(new api::CreateVisitorReply( - static_cast<const api::CreateVisitorCommand&>(cmd))); + auto msg = std::make_unique<api::CreateVisitorReply>(static_cast<const api::CreateVisitorCommand&>(cmd)); onDecodeReply(buf, *msg); vdslib::VisitorStatistics vs; @@ -625,11 +559,10 @@ ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, vs.setSecondPassBytesReturned(SH::getLong(buf)); msg->setVisitorStatistics(vs); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RequestBucketInfoCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const { const std::vector<document::BucketId>& buckets(msg.getBuckets()); buf.putInt(buckets.size()); @@ -663,7 +596,7 @@ ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const msg.reset(new api::RequestBucketInfoCommand(bucketSpace, distributor, state, SH::getString(buf))); } onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h index 02c4c0ce0b9..08cec601cce 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -4,13 +4,11 @@ #include <vespa/fastos/types.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/growablebytebuffer.h> -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { class SerializationHelper { @@ -60,8 +58,7 @@ public: return api::ReturnCode(result, message); } - static void putReturnCode(const api::ReturnCode& code, - vespalib::GrowableByteBuffer& buf) + static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) { buf.putInt(code.getResult()); buf.putString(code.getMessage()); @@ -77,18 +74,14 @@ public: return document::GlobalId(&buffer[0]); } - static void putGlobalId(const document::GlobalId& gid, - vespalib::GrowableByteBuffer& buf) + static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf) { buf.putShort(document::GlobalId::LENGTH); for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { buf.putByte(gid.get()[i]); } } - - static document::Document::UP getDocument( - document::ByteBuffer& buf, - const document::DocumentTypeRepo& repo) + static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo) { uint32_t size = getInt(buf); if (size == 0) { @@ -100,26 +93,7 @@ public: } } - static document::DocumentUpdate::UP getUpdate( - document::ByteBuffer& buf, - const document::DocumentTypeRepo& repo) - { - uint32_t size = getInt(buf); - if (size == 0) { - return document::DocumentUpdate::UP(); - } else { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); - buf.incPos(size); - return document::DocumentUpdate::UP( - new document::DocumentUpdate(repo, bbuf, - document::DocumentUpdate:: - SerializeVersion:: - SERIALIZE_42)); - } - } - - static void putDocument(document::Document* doc, - vespalib::GrowableByteBuffer& buf) + static void putDocument(document::Document* doc, vespalib::GrowableByteBuffer& buf) { if (doc) { vespalib::nbostream stream; @@ -131,21 +105,6 @@ public: } } - static void putUpdate(document::DocumentUpdate* update, - vespalib::GrowableByteBuffer& buf) - { - if (update) { - vespalib::nbostream stream; - update->serialize42(stream); - buf.putInt(stream.size()); - buf.putBytes(stream.peek(), stream.size()); - } else { - buf.putInt(0); - } - } - }; -} // mbusprot -} // storage - +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 33fca0f161a..f83188f7dd8 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -17,12 +17,12 @@ mbus::string StorageProtocol::NAME = "StorageProtocol"; StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentTypeRepo> repo, const documentapi::LoadTypeSet& loadTypes, - bool activateBucketSpaceSerialization) + bool configForcedBucketSpaceSerialization) : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), _serializer5_2(repo, loadTypes), _serializer6_0(repo, loadTypes), - _activateBucketSpaceSerialization(activateBucketSpaceSerialization) + _configForcedBucketSpaceSerialization(configForcedBucketSpaceSerialization) { } @@ -106,7 +106,7 @@ StorageProtocol::encode(const vespalib::Version& version, } else if (version < version5_2) { return encodeMessage(_serializer5_1, routable, message, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return encodeMessage(_serializer6_0, routable, message, version6_0, version); } else { if (version < version6_0) { @@ -184,7 +184,7 @@ StorageProtocol::decode(const vespalib::Version & version, } else if (version < version5_2) { return decodeMessage(_serializer5_1, data, type, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return decodeMessage(_serializer6_0, data, type, version6_0, version); } else { if (version < version6_0) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index d85e9d55d1a..56f271db1d0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -29,7 +29,7 @@ private: ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; ProtocolSerialization6_0 _serializer6_0; - bool _activateBucketSpaceSerialization; + bool _configForcedBucketSpaceSerialization; }; } diff --git a/storageserver/src/tests/testhelper.cpp b/storageserver/src/tests/testhelper.cpp index b245a6500bd..5e6a71b078f 100644 --- a/storageserver/src/tests/testhelper.cpp +++ b/storageserver/src/tests/testhelper.cpp @@ -96,7 +96,11 @@ vdstestlib::DirConfig getStandardConfig(bool storagenode) { // By default, need "old" behaviour of maxconcurrent config->set("maxconcurrentvisitors_fixed", "4"); config->set("maxconcurrentvisitors_variable", "0"); - config = &dc.addConfig("stor-visitordispatcher"); + dc.addConfig("stor-visitordispatcher"); + config = &dc.addConfig("bucketspaces"); + config->set("documenttype[1]"); + config->set("documenttype[0].name", "testdoctype1"); + config->set("documenttype[0].bucketspace", "default"); addFileConfig(dc, "documenttypes", "config-doctypes.cfg"); addStorageDistributionConfig(dc); return dc; diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index 92954b39f92..2df6c9c5691 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -6,6 +6,14 @@ fun:pthread_create@@GLIBC_2.2.5 } { + NPTL keeps a cache of thread stacks, and metadata for thread local storage is not freed for threads in that cache + Memcheck:Leak + fun:calloc + fun:allocate_dtv + fun:_dl_allocate_tls + fun:pthread_create@@GLIBC_2.2.5 +} +{ This is a bug in glibc. We can not suffer for that. Memcheck:Free fun:free @@ -35,6 +43,38 @@ fun:main } { + Bug in cppunit. This suppression is created on CentOS7. + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:allocate + fun:_S_create + fun:_S_construct<char const*> + fun:_S_construct_aux<char const*> + fun:_S_construct<char const*> + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7CppUnit10TestRunnerC1Ev + fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE + fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc + fun:main +} +{ + Bug in cppunit. This suppression is created on CentOS7. + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:allocate + fun:_S_create + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_S_construct_aux<char const*> + fun:_S_construct<char const*> + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7CppUnit10TestRunnerC1Ev + fun:_ZN7CppUnit14TextTestRunnerC1EPNS_9OutputterE + fun:_ZN10vdstestlib17CppUnitTestRunner3runEiPPKc + fun:main +} +{ RHEL6 strlen is eager and will read 16 bytes blocks. Memcheck:Cond fun:__strlen_sse42 diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java index c566d4fe4af..1cf19151b2e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzService.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.api; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; + import java.util.Objects; /** @@ -20,6 +22,15 @@ public class AthenzService implements AthenzIdentity { this(new AthenzDomain(domain), serviceName); } + public AthenzService(String fullName) { + AthenzIdentity identity = AthenzIdentities.from(fullName); + if (!(identity instanceof AthenzService)) { + throw new IllegalArgumentException(String.format("'%s' is not an Athenz service", fullName)); + } + AthenzService service = (AthenzService) identity; + this.domain = service.getDomain(); + this.serviceName = service.serviceName; + } @Override public AthenzDomain getDomain() { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index fb71ed65da1..ebff56a6f48 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -7,6 +7,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; +import com.yahoo.vespa.athenz.utils.SiaUtils; import javax.net.ssl.SSLContext; import java.io.File; @@ -42,8 +44,8 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), - getPrivateKeyFile(config.keyPathPrefix(), config.athenzDomain(), config.athenzService()), - getCertificateFile(config.keyPathPrefix(), config.athenzDomain(), config.athenzService()), + SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), + SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), new File(config.trustStorePath()), createScheduler()); } @@ -52,8 +54,8 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde Path siaPath, File trustStoreFile) { this(service, - getPrivateKeyFile(siaPath.toString(), service.getDomain().getName(), service.getName()), - getCertificateFile(siaPath.toString(), service.getDomain().getName(), service.getName()), + SiaUtils.getPrivateKeyFile(siaPath, service).toFile(), + SiaUtils.getCertificateFile(siaPath, service).toFile(), trustStoreFile, createScheduler()); } @@ -119,13 +121,6 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde } } - private static File getCertificateFile(String rootPath, String domain, String service) { - return Paths.get(rootPath, "certs", String.format("%s.%s.cert.pem", domain, service)).toFile(); - } - - private static File getPrivateKeyFile(String rootPath, String domain, String service) { - return Paths.get(rootPath, "keys", String.format("%s.%s.key.pem", domain, service)).toFile(); - } @Override public void deconstruct() { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index f879c2fa672..1504119d9cc 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.VespaUniqueInstanceIdEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.VespaUniqueInstanceIdEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import java.util.Base64; @@ -37,7 +36,7 @@ public class EntityBindingsMapper { entity.clusterIndex, entity.clusterId, entity.instance, entity.application, entity.tenant, entity.region, entity.environment); } - private static IdentityDocument toIdentityDocument(IdentityDocumentEntity entity) { + public static IdentityDocument toIdentityDocument(IdentityDocumentEntity entity) { return new IdentityDocument( toVespaUniqueInstanceId(entity.providerUniqueId), entity.configServerHostname, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocument.java deleted file mode 100644 index b2be9567258..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocument.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.time.Instant; -import java.util.Objects; -import java.util.Set; - -/** - * @author bjorncs - * @deprecated Use {@link IdentityDocumentEntity} instead. - */ -@Deprecated -@JsonIgnoreProperties(ignoreUnknown = true) -public class IdentityDocument { - - @JsonProperty("provider-unique-id") - public final ProviderUniqueId providerUniqueId; - @JsonProperty("configserver-hostname") - public final String configServerHostname; - @JsonProperty("instance-hostname") - public final String instanceHostname; - @JsonProperty("created-at") - public final Instant createdAt; - @JsonProperty("ip-addresses") - public final Set<String> ipAddresses; - - public IdentityDocument( - @JsonProperty("provider-unique-id") ProviderUniqueId providerUniqueId, - @JsonProperty("configserver-hostname") String configServerHostname, - @JsonProperty("instance-hostname") String instanceHostname, - @JsonProperty("created-at") Instant createdAt, - @JsonProperty("ip-addresses") Set<String> ipAddresses) { - this.providerUniqueId = providerUniqueId; - this.configServerHostname = configServerHostname; - this.instanceHostname = instanceHostname; - this.createdAt = createdAt; - this.ipAddresses = ipAddresses; - } - - - @Override - public String toString() { - return "IdentityDocument{" + - "providerUniqueId=" + providerUniqueId + - ", configServerHostname='" + configServerHostname + '\'' + - ", instanceHostname='" + instanceHostname + '\'' + - ", createdAt=" + createdAt + - ", ipAddresses=" + ipAddresses + - '}'; - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IdentityDocument that = (IdentityDocument) o; - return Objects.equals(providerUniqueId, that.providerUniqueId) && - Objects.equals(configServerHostname, that.configServerHostname) && - Objects.equals(instanceHostname, that.instanceHostname) && - Objects.equals(createdAt, that.createdAt) && - Objects.equals(ipAddresses, that.ipAddresses); - } - - @Override - public int hashCode() { - - return Objects.hash(providerUniqueId, configServerHostname, instanceHostname, createdAt, ipAddresses); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/ProviderUniqueId.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/ProviderUniqueId.java deleted file mode 100644 index eea469f282a..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/ProviderUniqueId.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; - -import java.util.Objects; - -/** - * @author bjorncs - * @deprecated Use {@link VespaUniqueInstanceIdEntity} instead. - */ -@Deprecated -public class ProviderUniqueId { - - @JsonProperty("tenant") - public final String tenant; - @JsonProperty("application") - public final String application; - @JsonProperty("environment") - public final String environment; - @JsonProperty("region") - public final String region; - @JsonProperty("instance") - public final String instance; - @JsonProperty("cluster-id") - public final String clusterId; - @JsonProperty("cluster-index") - public final int clusterIndex; - - public ProviderUniqueId(@JsonProperty("tenant") String tenant, - @JsonProperty("application") String application, - @JsonProperty("environment") String environment, - @JsonProperty("region") String region, - @JsonProperty("instance") String instance, - @JsonProperty("cluster-id") String clusterId, - @JsonProperty("cluster-index") int clusterIndex) { - this.tenant = tenant; - this.application = application; - this.environment = environment; - this.region = region; - this.instance = instance; - this.clusterId = clusterId; - this.clusterIndex = clusterIndex; - } - - public VespaUniqueInstanceId toVespaUniqueInstanceId() { - return new VespaUniqueInstanceId(clusterIndex, clusterId, instance, application, tenant, region, environment); - } - - public static ProviderUniqueId fromVespaUniqueInstanceId(VespaUniqueInstanceId instanceId) { - return new ProviderUniqueId( - instanceId.tenant(), instanceId.application(), instanceId.environment(), instanceId.region(), - instanceId.instance(), instanceId.clusterId(), instanceId.clusterIndex()); - } - - @Override - public String toString() { - return "ProviderUniqueId{" + - "tenant='" + tenant + '\'' + - ", application='" + application + '\'' + - ", environment='" + environment + '\'' + - ", region='" + region + '\'' + - ", instance='" + instance + '\'' + - ", clusterId='" + clusterId + '\'' + - ", clusterIndex=" + clusterIndex + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProviderUniqueId that = (ProviderUniqueId) o; - return clusterIndex == that.clusterIndex && - Objects.equals(tenant, that.tenant) && - Objects.equals(application, that.application) && - Objects.equals(environment, that.environment) && - Objects.equals(region, that.region) && - Objects.equals(instance, that.instance) && - Objects.equals(clusterId, that.clusterId); - } - - @Override - public int hashCode() { - return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocument.java deleted file mode 100644 index 20c3e236667..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocument.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URI; -import java.util.Base64; -import java.util.Objects; - -/** - * @author bjorncs - * @deprecated Use {@link SignedIdentityDocumentEntity} instead. - */ -@Deprecated -@JsonIgnoreProperties(ignoreUnknown = true) -public class SignedIdentityDocument { - - public static final int DEFAULT_KEY_VERSION = 0; - public static final int DEFAULT_DOCUMENT_VERSION = 1; - - private static final ObjectMapper mapper = createObjectMapper(); - - @JsonProperty("identity-document")public final String rawIdentityDocument; - @JsonIgnore public final IdentityDocument identityDocument; - @JsonProperty("signature") public final String signature; - @JsonProperty("signing-key-version") public final int signingKeyVersion; - @JsonProperty("provider-unique-id") public final String providerUniqueId; // String representation - @JsonProperty("dns-suffix") public final String dnsSuffix; - @JsonProperty("provider-service") public final String providerService; - @JsonProperty("zts-endpoint") public final URI ztsEndpoint; - @JsonProperty("document-version") public final int documentVersion; - - @JsonCreator - public SignedIdentityDocument(@JsonProperty("identity-document") String rawIdentityDocument, - @JsonProperty("signature") String signature, - @JsonProperty("signing-key-version") int signingKeyVersion, - @JsonProperty("provider-unique-id") String providerUniqueId, - @JsonProperty("dns-suffix") String dnsSuffix, - @JsonProperty("provider-service") String providerService, - @JsonProperty("zts-endpoint") URI ztsEndpoint, - @JsonProperty("document-version") int documentVersion) { - this.rawIdentityDocument = rawIdentityDocument; - this.identityDocument = parseIdentityDocument(rawIdentityDocument); - this.signature = signature; - this.signingKeyVersion = signingKeyVersion; - this.providerUniqueId = providerUniqueId; - this.dnsSuffix = dnsSuffix; - this.providerService = providerService; - this.ztsEndpoint = ztsEndpoint; - this.documentVersion = documentVersion; - } - - private static IdentityDocument parseIdentityDocument(String rawIdentityDocument) { - try { - return mapper.readValue(Base64.getDecoder().decode(rawIdentityDocument), IdentityDocument.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static ObjectMapper createObjectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - - @Override - public String toString() { - return "SignedIdentityDocument{" + - "rawIdentityDocument='" + rawIdentityDocument + '\'' + - ", identityDocument=" + identityDocument + - ", signature='" + signature + '\'' + - ", signingKeyVersion=" + signingKeyVersion + - ", documentVersion=" + documentVersion + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SignedIdentityDocument that = (SignedIdentityDocument) o; - return signingKeyVersion == that.signingKeyVersion && - documentVersion == that.documentVersion && - Objects.equals(rawIdentityDocument, that.rawIdentityDocument) && - Objects.equals(identityDocument, that.identityDocument) && - Objects.equals(signature, that.signature); - } - - @Override - public int hashCode() { - return Objects.hash(rawIdentityDocument, identityDocument, signature, signingKeyVersion, documentVersion); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index 2d6294e536c..e397b81ef9e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -20,9 +20,6 @@ import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) public class SignedIdentityDocumentEntity { - public static final int DEFAULT_KEY_VERSION = 0; - public static final int DEFAULT_DOCUMENT_VERSION = 1; - private static final ObjectMapper mapper = createObjectMapper(); @JsonProperty("identity-document")public final String rawIdentityDocument; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java index ae66899978e..bb9f512efe6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentials.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import javax.net.ssl.SSLContext; import java.security.KeyPair; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 554d50f296b..96e93ca419d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -3,28 +3,25 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.core.identity.IdentityConfig; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.tls.KeyAlgorithm; import com.yahoo.vespa.athenz.tls.KeyUtils; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; -import com.yahoo.vespa.athenz.tls.Pkcs10CsrBuilder; import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; -import com.yahoo.vespa.athenz.tls.SignatureAlgorithm; import com.yahoo.vespa.athenz.tls.SslContextBuilder; -import com.yahoo.vespa.athenz.tls.SubjectAlternativeName; import javax.net.ssl.SSLContext; -import javax.security.auth.x500.X500Principal; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.security.KeyPair; import java.security.PrivateKey; import java.security.cert.X509Certificate; -import java.util.Set; import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; -import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.IP_ADDRESS; /** * @author bjorncs @@ -52,39 +49,39 @@ class AthenzCredentialsService { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); String rawDocument = identityDocumentClient.getSignedIdentityDocument(); SignedIdentityDocument document = parseSignedIdentityDocument(rawDocument); - Pkcs10Csr csr = createCSR(identityConfig.domain(), - identityConfig.service(), - document.dnsSuffix, - document.providerUniqueId, - document.identityDocument.ipAddresses, - keyPair); + InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix()); + Pkcs10Csr csr = instanceCsrGenerator.generateCsr( + new AthenzService(identityConfig.domain(), identityConfig.service()), + document.providerUniqueId(), + document.identityDocument().ipAddresses(), + keyPair); InstanceRegisterInformation instanceRegisterInformation = - new InstanceRegisterInformation(document.providerService, + new InstanceRegisterInformation(document.providerService().getFullName(), identityConfig.domain(), identityConfig.service(), rawDocument, Pkcs10CsrUtils.toPem(csr)); InstanceIdentity instanceIdentity = ztsClient.sendInstanceRegisterRequest(instanceRegisterInformation, - document.ztsEndpoint); + document.ztsEndpoint()); return toAthenzCredentials(instanceIdentity, keyPair, document); } AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - Pkcs10Csr csr = createCSR(identityConfig.domain(), - identityConfig.service(), - document.dnsSuffix, - document.providerUniqueId, - document.identityDocument.ipAddresses, - newKeyPair); + InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix()); + Pkcs10Csr csr = instanceCsrGenerator.generateCsr( + new AthenzService(identityConfig.domain(), identityConfig.service()), + document.providerUniqueId(), + document.identityDocument().ipAddresses(), + newKeyPair); InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(Pkcs10CsrUtils.toPem(csr)); InstanceIdentity instanceIdentity = - ztsClient.sendInstanceRefreshRequest(document.providerService, + ztsClient.sendInstanceRefreshRequest(document.providerService().getFullName(), identityConfig.domain(), identityConfig.service(), - document.providerUniqueId, + document.providerUniqueId().asDottedString(), refreshInfo, - document.ztsEndpoint, + document.ztsEndpoint(), sslContext); return toAthenzCredentials(instanceIdentity, newKeyPair, document); } @@ -107,32 +104,9 @@ class AthenzCredentialsService { private static SignedIdentityDocument parseSignedIdentityDocument(String rawDocument) { try { - return mapper.readValue(rawDocument, SignedIdentityDocument.class); + return EntityBindingsMapper.toSignedIdentityDocument(mapper.readValue(rawDocument, SignedIdentityDocumentEntity.class)); } catch (IOException e) { throw new UncheckedIOException(e); } } - - private static Pkcs10Csr createCSR(String identityDomain, - String identityService, - String dnsSuffix, - String providerUniqueId, - Set<String> ipAddresses, - KeyPair keyPair) { - X500Principal subject = new X500Principal(String.format("CN=%s.%s", identityDomain, identityService)); - // Add SAN dnsname <service>.<domain-with-dashes>.<provider-dnsname-suffix> - // and SAN dnsname <provider-unique-instance-id>.instanceid.athenz.<provider-dnsname-suffix> - Pkcs10CsrBuilder pkcs10CsrBuilder = Pkcs10CsrBuilder.fromKeypair(subject, keyPair, SignatureAlgorithm.SHA256_WITH_RSA) - .addSubjectAlternativeName(String.format("%s.%s.%s", - identityService, - identityDomain.replace(".", "-"), - dnsSuffix)) - .addSubjectAlternativeName(String.format("%s.instanceid.athenz.%s", - providerUniqueId, - dnsSuffix)); - if(ipAddresses != null) { - ipAddresses.forEach(ipaddress -> pkcs10CsrBuilder.addSubjectAlternativeName(new SubjectAlternativeName(IP_ADDRESS, ipaddress))); - } - return pkcs10CsrBuilder.build(); - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 9c7f6cc8efb..db949929115 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -127,8 +127,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen X509Certificate roleCertificate = ztsClient.getRoleCertificate( new AthenzDomain(domain), role, - credentials.getIdentityDocument().dnsSuffix, - credentials.getIdentityDocument().ztsEndpoint, + credentials.getIdentityDocument().dnsSuffix(), + credentials.getIdentityDocument().ztsEndpoint(), identity, privateKey, credentials.getIdentitySslContext()); @@ -143,7 +143,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return ztsClient .getRoleToken( new AthenzDomain(domain), - credentials.getIdentityDocument().ztsEndpoint, + credentials.getIdentityDocument().ztsEndpoint(), credentials.getIdentitySslContext()) .getRawToken(); } @@ -154,7 +154,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen .getRoleToken( new AthenzDomain(domain), role, - credentials.getIdentityDocument().ztsEndpoint, + credentials.getIdentityDocument().ztsEndpoint(), credentials.getIdentitySslContext()) .getRawToken(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index 7de42bed1ce..90d1312c9f9 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -4,10 +4,11 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; @@ -80,12 +81,9 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { try (CloseableHttpResponse response = client.execute(request)) { String responseContent = EntityUtils.toString(response.getEntity()); if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { - com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument entity = - objectMapper.readValue( - responseContent, - com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocument.class); + SignedIdentityDocumentEntity entity = objectMapper.readValue(responseContent, SignedIdentityDocumentEntity.class); return new SignedIdentityDocument( - toEntityDocument(entity.identityDocument), + EntityBindingsMapper.toIdentityDocument(entity.identityDocument), entity.signature, entity.signingKeyVersion, VespaUniqueInstanceId.fromDottedString(entity.providerUniqueId), @@ -107,16 +105,6 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { } } - private static IdentityDocument toEntityDocument( - com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument identityDocument) { - return new IdentityDocument( - identityDocument.providerUniqueId.toVespaUniqueInstanceId(), - identityDocument.configServerHostname, - identityDocument.instanceHostname, - identityDocument.createdAt, - identityDocument.ipAddresses); - } - private static CloseableHttpClient createHttpClient(SSLContext sslContext, HostnameVerifier hostnameVerifier) { return HttpClientBuilder.create() diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java new file mode 100644 index 00000000000..adaafab4617 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -0,0 +1,39 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils; + +import com.yahoo.vespa.athenz.api.AthenzService; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Misc utility methods for SIA provided credentials + * + * @author bjorncs + */ +public class SiaUtils { + public static final Path DEFAULT_SIA_DIRECTORY = Paths.get("/var/lib/sia"); + + private SiaUtils() {} + + public static Path getPrivateKeyFile(AthenzService service) { + return getPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Path getPrivateKeyFile(Path root, AthenzService service) { + return root + .resolve("keys") + .resolve(String.format("%s.%s.key.pem", service.getDomainName(), service.getName())); + } + + public static Path getCertificateFile(AthenzService service) { + return getCertificateFile(DEFAULT_SIA_DIRECTORY, service); + } + + public static Path getCertificateFile(Path root, AthenzService service) { + return root + .resolve("certs") + .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); + } + +} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/api/bindings/IdentityDocumentTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/api/bindings/IdentityDocumentTest.java deleted file mode 100644 index cfc6e33b911..00000000000 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/api/bindings/IdentityDocumentTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.yahoo.vespa.athenz.api.bindings; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.google.common.collect.ImmutableSet; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.ProviderUniqueId; -import org.junit.Test; - -import java.io.IOException; -import java.time.Instant; - -import static org.junit.Assert.assertEquals; - -public class IdentityDocumentTest { - - @Test - public void test_serialization_deserialization() throws IOException { - IdentityDocument document = new IdentityDocument( - ProviderUniqueId.fromVespaUniqueInstanceId( - VespaUniqueInstanceId.fromDottedString("1.clusterId.instance.application.tenant.region.environment")), - "cfg.prod.xyz", - "foo.bar", - Instant.now(), - ImmutableSet.of("127.0.0.1", "::1")); - - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - String documentString = mapper.writeValueAsString(document); - IdentityDocument deserializedDocument = mapper.readValue(documentString, IdentityDocument.class); - assertEquals(document, deserializedDocument); - } -} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 98f0aa9b7ef..2e9b29f5327 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -1,10 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; import com.yahoo.vespa.athenz.tls.KeyStoreUtils; import org.junit.Rule; @@ -15,10 +23,12 @@ import org.mockito.stubbing.Answer; import java.io.File; import java.io.IOException; +import java.net.URI; import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import java.util.Collections; import java.util.Date; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; @@ -121,17 +131,19 @@ public class AthenzIdentityProviderImplTest { return file; } - private static String getIdentityDocument() { - return "{\n" + - " \"identity-document\": \"eyJwcm92aWRlci11bmlxdWUtaWQiOnsidGVuYW50IjoidGVuYW50IiwiYXBwbGljYXRpb24iOiJhcHBsaWNhdGlvbiIsImVudmlyb25tZW50IjoiZGV2IiwicmVnaW9uIjoidXMtbm9ydGgtMSIsImluc3RhbmNlIjoiZGVmYXVsdCIsImNsdXN0ZXItaWQiOiJkZWZhdWx0IiwiY2x1c3Rlci1pbmRleCI6MH0sImNvbmZpZ3NlcnZlci1ob3N0bmFtZSI6ImxvY2FsaG9zdCIsImluc3RhbmNlLWhvc3RuYW1lIjoieC55LmNvbSIsImNyZWF0ZWQtYXQiOjE1MDg3NDgyODUuNzQyMDAwMDAwfQ==\",\n" + - " \"signature\": \"kkEJB/98cy1FeXxzSjtvGH2a6BFgZu/9/kzCcAqRMZjENxnw5jyO1/bjZVzw2Sz4YHPsWSx2uxb32hiQ0U8rMP0zfA9nERIalSP0jB/hMU8laezGhdpk6VKZPJRC6YKAB9Bsv2qUIfMsSxkMqf66GUvjZAGaYsnNa2yHc1jIYHOGMeJO+HNPYJjGv26xPfAOPIKQzs3RmKrc3FoweTCsIwm5oblqekdJvVWYe0obwlOSB5uwc1zpq3Ie1QBFtJRuCGMVHg1pDPxXKBHLClGIrEvzLmICy6IRdHszSO5qiwujUD7sbrbM0sB/u0cYucxbcsGRUmBvme3UAw2mW9POVQ==\",\n" + - " \"signing-key-version\": 0,\n" + - " \"provider-unique-id\": \"tenant.application.dev.us-north-1.default.default.0\",\n" + - " \"dns-suffix\": \"dnsSuffix\",\n" + - " \"provider-service\": \"service\",\n" + - " \"zts-endpoint\": \"localhost/zts\", \n" + - " \"document-version\": 1\n" + - "}"; - + private static String getIdentityDocument() throws JsonProcessingException { + VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev"); + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + new IdentityDocument(instanceId, "localhost", "x.y.com", Instant.EPOCH, Collections.emptySet()), + "dummysignature", + 0, + instanceId, + "dev-us-north-1.vespa.cloud", + new AthenzService("vespa.vespa.provider_dev_us-north-1"), + URI.create("https://zts:4443/zts/v1"), + 1); + + return new ObjectMapper().registerModule(new JavaTimeModule()) + .writeValueAsString(EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument)); } } diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh index f81c2d9ada3..e61a3d8c8b1 100755 --- a/vespabase/src/start-cbinaries.sh +++ b/vespabase/src/start-cbinaries.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # BEGIN environment bootstrap section diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java index 7a59be49458..4390f70cac0 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java @@ -7,7 +7,6 @@ import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.Raw; import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.templates.Context; import com.yahoo.text.XML; import java.io.IOException; @@ -38,7 +37,7 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void error(Context context, Writer writer) throws IOException { + public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { // Error shouldn't be handled by this template, but rather // delegated to the searcher } @@ -55,7 +54,7 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void header(Context context, Writer writer) throws IOException { + public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { if (wrapXml) { // XML wrapping should only be used for default field rendering writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); @@ -64,14 +63,14 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void footer(Context context, Writer writer) throws IOException { + public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { if (wrapXml) { writer.write("</result>\n"); } } @Override - public void hit(Context context, Writer writer) throws IOException { + public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { DocumentHit hit = (DocumentHit)context.get("hit"); Document doc = hit.getDocument(); // Assume field existence has been checked before we ever get here. @@ -88,11 +87,11 @@ public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTempl } @Override - public void hitFooter(Context context, Writer writer) throws IOException { + public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } @Override - public void noHits(Context context, Writer writer) throws IOException { + public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java index 25ee0ff5d03..b16f39800ef 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java @@ -6,7 +6,6 @@ import com.yahoo.search.Result; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.HitGroup; -import com.yahoo.prelude.templates.Context; import com.yahoo.search.result.Hit; import com.yahoo.text.XML; @@ -55,7 +54,7 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void error(Context context, Writer writer) throws IOException { + public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("<errors>\n"); // If the error contains no error hits, use a single error with the main // code and description. Otherwise, use the error hits explicitly @@ -72,7 +71,7 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void header(Context context, Writer writer) throws IOException { + public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); writer.write("<result>\n"); HitGroup rootGroup = ((Result) context.get("result")).hits(); @@ -82,12 +81,12 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void footer(Context context, Writer writer) throws IOException { + public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { writer.write("</result>\n"); } @Override - public void hit(Context context, Writer writer) throws IOException { + public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { Hit hit = (Hit)context.get("hit"); if (hit instanceof DocumentHit) { DocumentHit docHit = (DocumentHit) hit; @@ -110,11 +109,11 @@ public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplat } @Override - public void hitFooter(Context context, Writer writer) throws IOException { + public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } @Override - public void noHits(Context context, Writer writer) throws IOException { + public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 5f1c96d000f..5f49dd5ddf8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -40,6 +40,8 @@ import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL; * The implementation is based on the code from V2, but the object model is rewritten to simplify the logic and * avoid using a threadpool that has no effect with all the extra that comes with it. V2 has one instance per thread * on the client, while this is one instance for all threads. + * + * @author dybis */ class ClientFeederV3 { @@ -109,7 +111,7 @@ class ClientFeederV3 { ongoingRequests.incrementAndGet(); try { FeederSettings feederSettings = new FeederSettings(request); - /** + /* * The gateway handle overload from clients in different ways. * * If the backend is overloaded, but not the gateway, it will fill the backend, messagebus throttler @@ -132,7 +134,7 @@ class ClientFeederV3 { } InputStream inputStream = StreamReaderV3.unzipStreamIfNeeded(request); - final BlockingQueue<OperationStatus> replies = new LinkedBlockingQueue<>(); + BlockingQueue<OperationStatus> replies = new LinkedBlockingQueue<>(); try { feed(feederSettings, inputStream, replies, threadsAvailableForFeeding); synchronized (monitor) { @@ -148,11 +150,7 @@ class ClientFeederV3 { log.log(LogLevel.WARNING, "Unhandled exception while feeding: " + Exceptions.toMessageString(e), e); } finally { - try { - replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, false, null)); - } catch (InterruptedException e) { - // NOP, we are already exiting the thread - } + replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, false, null)); } return new FeedResponse(200, replies, 3 /* protocol version */, clientId, outstandingOperations.get(), hostName); } finally { @@ -171,7 +169,7 @@ class ClientFeederV3 { private Optional<DocumentOperationMessageV3> pullMessageFromRequest( FeederSettings settings, InputStream requestInputStream, BlockingQueue<OperationStatus> repliesFromOldMessages) { while (true) { - final Optional<String> operationId; + Optional<String> operationId; try { operationId = streamReaderV3.getNextOperationId(requestInputStream); } catch (IOException ioe) { @@ -183,9 +181,10 @@ class ClientFeederV3 { if (! operationId.isPresent()) { return Optional.empty(); } - final DocumentOperationMessageV3 msg; + + DocumentOperationMessageV3 message; try { - msg = getNextMessage(operationId.get(), requestInputStream, settings); + message = getNextMessage(operationId.get(), requestInputStream, settings); } catch (Exception e) { if (log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); @@ -195,8 +194,9 @@ class ClientFeederV3 { continue; } - setRoute(msg, settings); - return Optional.of(msg); + if (message != null) + setRoute(message, settings); + return Optional.ofNullable(message); } } @@ -235,7 +235,7 @@ class ClientFeederV3 { } setMessageParameters(msg.get(), settings); - final Result result; + Result result; try { result = sendMessage(settings, msg.get(), threadsAvailableForFeeding); @@ -264,8 +264,8 @@ class ClientFeederV3 { } } - private OperationStatus createOperationStatus(String id, String message, ErrorCode code, boolean isConditionNotMet, Message msg) - throws InterruptedException { + private OperationStatus createOperationStatus(String id, String message, + ErrorCode code, boolean isConditionNotMet, Message msg) { String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0 ? msg.getTrace().toString() : ""; @@ -273,6 +273,7 @@ class ClientFeederV3 { } // protected for mocking + /** Returns the next message in the stream, or null if none */ protected DocumentOperationMessageV3 getNextMessage( String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings); @@ -285,14 +286,14 @@ class ClientFeederV3 { null); } - DocumentOperationMessageV3 msg = DocumentOperationMessageV3.create(operation, operationId, metric); - if (msg == null) { + DocumentOperationMessageV3 message = DocumentOperationMessageV3.create(operation, operationId, metric); + if (message == null) { // typical end of feed return null; } metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); - log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.getOperationId()); - return msg; + log(LogLevel.DEBUG, "Successfully deserialized document id: ", message.getOperationId()); + return message; } private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java index 2424ce596a3..9d6c8c2feac 100755 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java @@ -17,11 +17,9 @@ import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; -import com.yahoo.prelude.templates.SearchRendererAdaptor; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; @@ -740,7 +738,7 @@ public class GetSearcherTestCase { assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); ByteArrayOutputStream stream = new ByteArrayOutputStream(); - SearchRendererAdaptor.callRender(stream, result); + com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); stream.flush(); byte[] resultBytes = stream.toByteArray(); @@ -769,7 +767,7 @@ public class GetSearcherTestCase { assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); ByteArrayOutputStream stream = new ByteArrayOutputStream(); - SearchRendererAdaptor.callRender(stream, result); + com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); stream.flush(); byte[] resultBytes = stream.toByteArray(); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java index 1c855455a37..d42faf418a1 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java @@ -33,7 +33,6 @@ public class BucketStatsRetriever { private final MessageBusSyncSession session; private final MessageBusDocumentAccess documentAccess; - private final String route; public BucketStatsRetriever( DocumentAccessFactory documentAccessFactory, @@ -42,7 +41,7 @@ public class BucketStatsRetriever { registerShutdownHook(registrar); this.documentAccess = documentAccessFactory.createDocumentAccess(); this.session = documentAccess.createSyncSession(new SyncParameters.Builder().build()); - this.route = route; + this.session.setRoute(route); } private void registerShutdownHook(ShutdownHookRegistrar registrar) { @@ -102,19 +101,10 @@ public class BucketStatsRetriever { private <T extends Reply> T sendMessage(DocumentMessage msg, Class<T> expectedReply) throws BucketStatsException { - setRoute(msg, route); Reply reply = session.syncSend(msg); return validateReply(reply, expectedReply); } - private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException { - try { - msg.setRoute(Route.parse(route)); - } catch (Exception e) { - throw new BucketStatsException(String.format("Invalid route: '%s'.", route)); - } - } - private static <T extends Reply> T validateReply(Reply reply, Class<T> type) throws BucketStatsException { if (reply.hasErrors()) { throw new BucketStatsException(makeErrorMessage(reply)); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java index a34e3a73205..475547546d3 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java @@ -20,6 +20,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -127,12 +128,8 @@ public class BucketStatsRetrieverTest { BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {}); retriever.retrieveBucketList(new BucketId(0), bucketSpace); - verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() { - @Override - public boolean matches(Object o) { - return ((Message) o).getRoute().equals(Route.parse(route)); - } - })); + // Route is set at session-level, not per message sent. + verify(mockedSession).setRoute(eq(route)); } private BucketStatsRetriever createRetriever() { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java new file mode 100644 index 00000000000..439aac5578a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java @@ -0,0 +1,90 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.yahoo.tensor.evaluation.MapEvaluationContext; +import com.yahoo.tensor.evaluation.VariableTensor; +import com.yahoo.tensor.functions.ConstantTensor; +import com.yahoo.tensor.functions.Join; +import com.yahoo.tensor.functions.Reduce; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +/** + * Microbenchmark of a "dot product" of two mapped rank 2 tensors + * + * @author bratseth + */ +public class MatrixDotProductBenchmark { + + private final static Random random = new Random(); + + public double benchmark(int iterations, List<Tensor> modelMatrixes, TensorType.Dimension.Type dimensionType) { + Tensor queryMatrix = matrix(1, 20, dimensionType).get(0); + dotProduct(queryMatrix, modelMatrixes, Math.max(iterations/10, 10)); // warmup + System.gc(); + long startTime = System.currentTimeMillis(); + dotProduct(queryMatrix, modelMatrixes, iterations); + long totalTime = System.currentTimeMillis() - startTime; + return (double)totalTime / (double)iterations; + } + + private double dotProduct(Tensor tensor, List<Tensor> tensors, int iterations) { + double result = 0; + for (int i = 0 ; i < iterations; i++) + result = dotProduct(tensor, tensors); + return result; + } + + private double dotProduct(Tensor tensor, List<Tensor> tensors) { + double largest = Double.MIN_VALUE; + TensorFunction dotProductFunction = new Reduce(new Join(new ConstantTensor(tensor), + new VariableTensor("argument"), (a, b) -> a * b), + Reduce.Aggregator.sum).toPrimitive(); + MapEvaluationContext context = new MapEvaluationContext(); + + for (Tensor tensorElement : tensors) { // tensors.size() = 1 for larger tensor + context.put("argument", tensorElement); + double dotProduct = dotProductFunction.evaluate(context).asDouble(); + if (dotProduct > largest) { + largest = dotProduct; + } + } + return largest; + } + + private static List<Tensor> matrix(int dimension1Size, int dimension2Size, TensorType.Dimension.Type dimensionType) { + TensorType.Builder typeBuilder = new TensorType.Builder(); + addDimension(typeBuilder, "i", dimensionType, dimension1Size); + addDimension(typeBuilder, "j", dimensionType, dimension2Size); + Tensor.Builder builder = Tensor.Builder.of(typeBuilder.build()); + for (int i = 0; i < dimension1Size; i++) { + for (int j = 0; j < dimension2Size; j++) { + builder.cell() + .label("i", String.valueOf("label" + i)) + .label("j", String.valueOf("label" + j)) + .value(random.nextDouble()); + } + } + return Collections.singletonList(builder.build()); + } + + private static void addDimension(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) { + switch (type) { + case mapped: builder.mapped(name); break; + case indexedUnbound: builder.indexed(name); break; + case indexedBound: builder.indexed(name, size); break; + default: throw new IllegalArgumentException("Dimension type " + type + " not supported"); + } + } + + public static void main(String[] args) { + double time = new MatrixDotProductBenchmark().benchmark(10000, matrix(10, 55, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped); + System.out.printf("Matrixes, 10*55 size matrixes. Time per sum(join): %1$8.3f ms\n", time); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java index abdb3071bf7..7b856dde2d5 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java @@ -107,26 +107,26 @@ public class TensorFunctionBenchmark { double time = 0; // ---------------- Mapped with extra space (sidesteps current special-case optimizations): - // 9.9 ms + // 7.8 ms time = new TensorFunctionBenchmark().benchmark(1000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true); System.out.printf("Mapped vectors, x space time per join: %1$8.3f ms\n", time); - // 10.5 ms + // 7.7 ms time = new TensorFunctionBenchmark().benchmark(1000, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true); System.out.printf("Mapped matrix, x space time per join: %1$8.3f ms\n", time); // ---------------- Mapped: - // 2.6 ms + // 2.1 ms time = new TensorFunctionBenchmark().benchmark(5000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false); System.out.printf("Mapped vectors, time per join: %1$8.3f ms\n", time); - // 6.8 ms + // 7.0 ms time = new TensorFunctionBenchmark().benchmark(1000, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false); System.out.printf("Mapped matrix, time per join: %1$8.3f ms\n", time); // ---------------- Indexed (unbound) with extra space (sidesteps current special-case optimizations): - // 30 ms + // 14.5 ms time = new TensorFunctionBenchmark().benchmark(500, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true); System.out.printf("Indexed vectors, x space time per join: %1$8.3f ms\n", time); - // 27 ms + // 8.9 ms time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true); System.out.printf("Indexed matrix, x space time per join: %1$8.3f ms\n", time); @@ -134,15 +134,15 @@ public class TensorFunctionBenchmark { // 0.14 ms time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false); System.out.printf("Indexed unbound vectors, time per join: %1$8.3f ms\n", time); - // 0.14 ms + // 0.44 ms time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false); System.out.printf("Indexed unbound matrix, time per join: %1$8.3f ms\n", time); // ---------------- Indexed bound: - // 0.14 ms + // 0.32 ms time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false); System.out.printf("Indexed bound vectors, time per join: %1$8.3f ms\n", time); - // 0.14 ms + // 0.44 ms time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false); System.out.printf("Indexed bound matrix, time per join: %1$8.3f ms\n", time); } diff --git a/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp b/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp index 057d45e8ef5..5c84790a669 100644 --- a/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp +++ b/vespalib/src/apps/vespa-detect-hostname/detect_hostname.cpp @@ -4,6 +4,7 @@ #include <stdlib.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/stringfmt.h> #include <set> using vespalib::SocketAddress; @@ -22,14 +23,17 @@ vespalib::string get_hostname() { return SocketAddress::normalize(&result[0]); } -bool check(const vespalib::string &name, const std::set<vespalib::string> &ip_set) { +bool check(const vespalib::string &name, const std::set<vespalib::string> &ip_set, vespalib::string &error_msg) { auto addr_list = SocketAddress::resolve(80, name.c_str()); if (addr_list.empty()) { + error_msg = vespalib::make_string("hostname '%s' could not be resolved", name.c_str()); return false; } for (const SocketAddress &addr: addr_list) { vespalib::string ip_addr = addr.ip_address(); if (ip_set.count(ip_addr) == 0) { + error_msg = vespalib::make_string("hostname '%s' resolves to ip address not owned by this host (%s)", + name.c_str(), ip_addr.c_str()); return false; } } @@ -38,14 +42,21 @@ bool check(const vespalib::string &name, const std::set<vespalib::string> &ip_se int main(int, char **) { auto my_ip_set = make_ip_set(); - std::vector<vespalib::string> list({get_hostname(), "localhost", "127.0.0.1", "::1"}); - for (const vespalib::string &name: list) { - if (check(name, my_ip_set)) { - fprintf(stdout, "%s\n", name.c_str()); - return 0; - } + vespalib::string my_hostname = get_hostname(); + vespalib::string my_hostname_error; + vespalib::string localhost = "localhost"; + vespalib::string localhost_error; + if (check(my_hostname, my_ip_set, my_hostname_error)) { + fprintf(stdout, "%s\n", my_hostname.c_str()); + } else if (check(localhost, my_ip_set, localhost_error)) { + fprintf(stdout, "%s\n", localhost.c_str()); + } else { + fprintf(stderr, "FATAL: hostname detection failed\n"); + fprintf(stderr, " INFO: canonical hostname (from gethostname/getaddrinfo): %s\n", my_hostname.c_str()); + fprintf(stderr, " ERROR: %s\n", my_hostname_error.c_str()); + fprintf(stderr, " INFO: falling back to local hostname: %s\n", localhost.c_str()); + fprintf(stderr, " ERROR: %s\n", localhost_error.c_str()); + return 1; } - fprintf(stderr, "FATAL: hostname detection failed\n"); - // XXX we should explain why it failed - return 1; + return 0; } diff --git a/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp b/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp index c97927884f8..88c2d07dd29 100644 --- a/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp +++ b/vespalib/src/apps/vespa-validate-hostname/validate_hostname.cpp @@ -25,19 +25,6 @@ vespalib::string normalize(const vespalib::string &hostname) { return canon_name; } -void check_reverse(const vespalib::string &hostname, const SocketAddress &addr) { - std::set<vespalib::string> seen({hostname}); - vespalib::string reverse = addr.reverse_lookup(); - for (size_t i = 0; !reverse.empty() && (i < 10); ++i) { - if (seen.count(reverse) == 0) { - seen.insert(reverse); - fprintf(stderr, "warning: hostname validation: found conflicting reverse lookup: '%s' -> %s -> '%s'\n", - hostname.c_str(), addr.ip_address().c_str(), reverse.c_str()); - } - reverse = addr.reverse_lookup(); - } -} - int usage(const char *self) { fprintf(stderr, "usage: %s <hostname>\n", self); return 1; @@ -62,8 +49,6 @@ int main(int argc, char **argv) { valid = false; fprintf(stderr, "FATAL: hostname validation failed: '%s' resolves to ip address not owned by this host (%s)\n", hostname.c_str(), ip_addr.c_str()); - } else { - check_reverse(hostname, addr); } } return valid ? 0 : 1; diff --git a/vespalib/src/tests/guard/guard_test.cpp b/vespalib/src/tests/guard/guard_test.cpp index a9d5d5f894c..9889f466edb 100644 --- a/vespalib/src/tests/guard/guard_test.cpp +++ b/vespalib/src/tests/guard/guard_test.cpp @@ -35,7 +35,8 @@ Test::testFilePointer() FilePointer file(fopen("filept.txt", "r")); EXPECT_TRUE(file.valid()); char tmp[128]; - fgets(tmp, sizeof(tmp), file); + char *fgetsres = fgets(tmp, sizeof(tmp), file); + ASSERT_EQUAL(tmp, fgetsres); EXPECT_TRUE(strcmp(tmp, "Hello") == 0); } { @@ -57,7 +58,8 @@ Test::testFilePointer() file.reset(fopen("filept.txt", "r")); EXPECT_TRUE(file.valid()); char tmp[128]; - fgets(tmp, sizeof(tmp), file.fp()); + char *fgetsres = fgets(tmp, sizeof(tmp), file.fp()); + ASSERT_EQUAL(tmp, fgetsres); EXPECT_TRUE(strcmp(tmp, "World") == 0); FILE *ref = file.fp(); diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index 94e214e9fb9..366111cad0d 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -434,6 +434,45 @@ TEST("test that for_each member works as std::for_each") { TEST_DO(verify_sum(m, expected_sum)); } +namespace { + +class WrappedKey +{ + std::unique_ptr<const int> _key; +public: + WrappedKey() : _key() { } + WrappedKey(int key) : _key(std::make_unique<const int>(key)) { } + size_t hash() const { return vespalib::hash<int>()(*_key); } + bool operator==(const WrappedKey &rhs) const { return *_key == *rhs._key; } +}; + +} + +TEST("test that hash map can have non-copyable key") +{ + hash_map<WrappedKey, int> m; + EXPECT_TRUE(m.insert(std::make_pair(WrappedKey(4), 5)).second); + WrappedKey testKey(4); + ASSERT_TRUE(m.find(testKey) != m.end()); + EXPECT_EQUAL(5, m.find(testKey)->second); +} + +TEST("test that hash map can have non-copyable value") +{ + hash_map<int, std::unique_ptr<int>> m; + EXPECT_TRUE(m.insert(std::make_pair(4, std::make_unique<int>(5))).second); + EXPECT_TRUE(m[4]); + EXPECT_EQUAL(5, *m[4]); +} + +TEST("test that hash set can have non-copyable key") +{ + hash_set<WrappedKey> m; + EXPECT_TRUE(m.insert(WrappedKey(4)).second); + WrappedKey testKey(4); + ASSERT_TRUE(m.find(testKey) != m.end()); +} + using IntHashSet = hash_set<int>; TEST("test hash set initializer list - empty") diff --git a/vespalib/src/tests/testapp-debug/testapp-debug.cpp b/vespalib/src/tests/testapp-debug/testapp-debug.cpp index 8c75a104cd8..0083200ac51 100644 --- a/vespalib/src/tests/testapp-debug/testapp-debug.cpp +++ b/vespalib/src/tests/testapp-debug/testapp-debug.cpp @@ -4,8 +4,12 @@ using namespace vespalib; TEST_MAIN() { - system("./vespalib_debug_test_app"); - system("diff lhs.out rhs.out > diff.out"); + int status = system("./vespalib_debug_test_app"); + ASSERT_FALSE(WIFSIGNALED(status)); + EXPECT_NOT_EQUAL(0, WEXITSTATUS(status)); + status = system("diff lhs.out rhs.out > diff.out"); + ASSERT_FALSE(WIFSIGNALED(status)); + EXPECT_NOT_EQUAL(0, WEXITSTATUS(status)); std::string diff_cmd("diff diff.out "); diff --git a/vespalib/src/tests/testapp-state/testapp-state.cpp b/vespalib/src/tests/testapp-state/testapp-state.cpp index eda67a15d60..683476a5525 100644 --- a/vespalib/src/tests/testapp-state/testapp-state.cpp +++ b/vespalib/src/tests/testapp-state/testapp-state.cpp @@ -4,8 +4,12 @@ using namespace vespalib; TEST_MAIN() { - system("./vespalib_state_test_app > out.txt 2>&1 out.txt"); - system("cat out.txt | grep STATE | sed 's/([^)].*\\//(/' > actual.txt"); + int status = system("./vespalib_state_test_app > out.txt 2>&1 out.txt"); + ASSERT_FALSE(WIFSIGNALED(status)); + EXPECT_NOT_EQUAL(0, WEXITSTATUS(status)); + status = system("cat out.txt | grep STATE | sed 's/([^)].*\\//(/' > actual.txt"); + ASSERT_FALSE(WIFSIGNALED(status)); + EXPECT_EQUAL(0, WEXITSTATUS(status)); std::string diff_cmd("diff -u actual.txt "); diff_cmd += TEST_PATH("expect.txt"); diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index 18bf5289f0d..72b494e2479 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -422,7 +422,7 @@ JsonDecoder::decodeNumber(Inserter &inserter) switch (c) { case '+': case '-': case '.': case 'e': case 'E': isLong = false; - //@fallthrough@ + [[fallthrough]]; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': value.push_back(c); diff --git a/vespalib/src/vespa/vespalib/net/selector.cpp b/vespalib/src/vespa/vespalib/net/selector.cpp index 6fad2e71444..e59638b6144 100644 --- a/vespalib/src/vespa/vespalib/net/selector.cpp +++ b/vespalib/src/vespa/vespalib/net/selector.cpp @@ -45,14 +45,14 @@ void WakeupPipe::write_token() { char token = 'T'; - write(_pipe[1], &token, 1); + [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1); } void WakeupPipe::read_tokens() { char token_trash[128]; - read(_pipe[0], token_trash, sizeof(token_trash)); + [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash)); } //----------------------------------------------------------------------------- diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index 0225b788e68..ea9684efe01 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -72,7 +72,7 @@ nbostream::operator = (const nbostream & rhs) { return *this; } -nbostream::~nbostream() { } +nbostream::~nbostream() = default; void nbostream::fail(State s) { diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index f2431a73f28..6d6498f8e78 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -36,6 +36,7 @@ public: size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } insert_result insert(const value_type & value) { return _ht.insert(value); } + insert_result insert(value_type &&value) { return _ht.insert(std::move(value)); } template <typename InputIt> void insert(InputIt first, InputIt last); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h index bb16932a990..c4ccc662787 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h @@ -37,6 +37,7 @@ public: size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } insert_result insert(const K & value); + insert_result insert(K &&value); template<typename InputIt> void insert(InputIt first, InputIt last); void erase(const K & key); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp index bd323b2c860..cf6341218f1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp @@ -74,6 +74,12 @@ hash_set<K, H, EQ, M>::insert(const K & value) { return _ht.insert(value); } +template<typename K, typename H, typename EQ, typename M> +typename hash_set<K, H, EQ, M>::insert_result +hash_set<K, H, EQ, M>::insert(K &&value) { + return _ht.insert(std::move(value)); +} + } #define VESPALIB_HASH_SET_INSTANTIATE(K) \ diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index dff744a0a41..01d0f91dfa3 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -186,6 +186,7 @@ struct MMapLimitAndAlignmentHash { }; using AutoAllocatorsMap = std::unordered_map<MMapLimitAndAlignment, AutoAllocator::UP, MMapLimitAndAlignmentHash>; +using AutoAllocatorsMapWithDefault = std::pair<AutoAllocatorsMap, alloc::MemoryAllocator *>; void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) { for (size_t alignment : {0,0x200, 0x400, 0x1000}) { @@ -197,7 +198,8 @@ void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) { } } -AutoAllocatorsMap createAutoAllocators() { +AutoAllocatorsMap +createAutoAllocators() { AutoAllocatorsMap map; map.reserve(3*5); for (size_t pages : {1,2,4,8,16}) { @@ -207,7 +209,30 @@ AutoAllocatorsMap createAutoAllocators() { return map; } -AutoAllocatorsMap _G_availableAutoAllocators = createAutoAllocators(); +MemoryAllocator & +getAutoAllocator(AutoAllocatorsMap & map, size_t mmapLimit, size_t alignment) { + MMapLimitAndAlignment key(mmapLimit, alignment); + auto found = map.find(key); + if (found == map.end()) { + throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment)); + } + return *(found->second); +} + +MemoryAllocator & +getDefaultAutoAllocator(AutoAllocatorsMap & map) { + return getAutoAllocator(map, 1 * MemoryAllocator::HUGEPAGE_SIZE, 0); +} + +AutoAllocatorsMapWithDefault +createAutoAllocatorsWithDefault() { + AutoAllocatorsMapWithDefault tmp(createAutoAllocators(), nullptr); + tmp.second = &getDefaultAutoAllocator(tmp.first); + return tmp; +} + + +AutoAllocatorsMapWithDefault _G_availableAutoAllocators = createAutoAllocatorsWithDefault(); alloc::HeapAllocator _G_heapAllocatorDefault; alloc::AlignedHeapAllocator _G_4KalignedHeapAllocator(1024); alloc::AlignedHeapAllocator _G_1KalignedHeapAllocator(4096); @@ -216,11 +241,13 @@ alloc::MMapAllocator _G_mmapAllocatorDefault; } -MemoryAllocator & HeapAllocator::getDefault() { +MemoryAllocator & +HeapAllocator::getDefault() { return _G_heapAllocatorDefault; } -MemoryAllocator & AlignedHeapAllocator::get4K() { +MemoryAllocator & +AlignedHeapAllocator::get4K() { return _G_4KalignedHeapAllocator; } @@ -228,25 +255,23 @@ MemoryAllocator & AlignedHeapAllocator::get1K() { return _G_1KalignedHeapAllocator; } -MemoryAllocator & AlignedHeapAllocator::get512B() { +MemoryAllocator & +AlignedHeapAllocator::get512B() { return _G_512BalignedHeapAllocator; } -MemoryAllocator & MMapAllocator::getDefault() { +MemoryAllocator & +MMapAllocator::getDefault() { return _G_mmapAllocatorDefault; } -MemoryAllocator & AutoAllocator::getDefault() { - return getAllocator(1 * MemoryAllocator::HUGEPAGE_SIZE, 0); +MemoryAllocator & +AutoAllocator::getDefault() { + return *_G_availableAutoAllocators.second; } MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment) { - MMapLimitAndAlignment key(mmapLimit, alignment); - auto found = _G_availableAutoAllocators.find(key); - if (found == _G_availableAutoAllocators.end()) { - throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment)); - } - return *(found->second); + return getAutoAllocator(_G_availableAutoAllocators.first, mmapLimit, alignment); } MemoryAllocator::PtrAndSize @@ -466,6 +491,12 @@ Alloc::allocMMap(size_t sz) } Alloc +Alloc::alloc() +{ + return Alloc(&AutoAllocator::getDefault()); +} + +Alloc Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment) { return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz); diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 2c3de92c58e..b52cace45a5 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -88,7 +88,7 @@ public: std::swap(_allocator, rhs._allocator); } Alloc create(size_t sz) const { - return Alloc(_allocator, sz); + return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz); } static Alloc allocAlignedHeap(size_t sz, size_t alignment); @@ -98,9 +98,11 @@ public: * Optional alignment is assumed to be <= system page size, since mmap * is always used when size is above limit. */ - static Alloc alloc(size_t sz=0, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); + static Alloc alloc(size_t sz, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); + static Alloc alloc(); private: Alloc(const MemoryAllocator * allocator, size_t sz) : _alloc(allocator->alloc(sz)), _allocator(allocator) { } + Alloc(const MemoryAllocator * allocator) : _alloc(nullptr, 0), _allocator(allocator) { } void clear() { _alloc.first = nullptr; _alloc.second = 0; diff --git a/vespalib/src/vespa/vespalib/util/bobhash.h b/vespalib/src/vespa/vespalib/util/bobhash.h index 2aff09929b2..60cbe2cbca3 100644 --- a/vespalib/src/vespa/vespalib/util/bobhash.h +++ b/vespalib/src/vespa/vespalib/util/bobhash.h @@ -128,18 +128,18 @@ public: c += length; switch(len) /* all the case statements fall through */ { - case 11: c += (static_cast<uint32_t>(k[10]) << 24); //@fallthrough@ - case 10: c += (static_cast<uint32_t>(k[9]) << 16); //@fallthrough@ - case 9 : c += (static_cast<uint32_t>(k[8]) << 8); //@fallthrough@ + case 11: c += (static_cast<uint32_t>(k[10]) << 24); [[fallthrough]]; + case 10: c += (static_cast<uint32_t>(k[9]) << 16); [[fallthrough]]; + case 9 : c += (static_cast<uint32_t>(k[8]) << 8); [[fallthrough]]; /* the first byte of c is reserved for the length */ - case 8 : b += (static_cast<uint32_t>(k[7]) << 24); //@fallthrough@ - case 7 : b += (static_cast<uint32_t>(k[6]) << 16); //@fallthrough@ - case 6 : b += (static_cast<uint32_t>(k[5]) << 8); //@fallthrough@ - case 5 : b += k[4]; //@fallthrough@ - case 4 : a += (static_cast<uint32_t>(k[3]) << 24); //@fallthrough@ - case 3 : a += (static_cast<uint32_t>(k[2]) << 16); //@fallthrough@ - case 2 : a += (static_cast<uint32_t>(k[1]) << 8); //@fallthrough@ - case 1 : a += k[0]; //@fallthrough@ + case 8 : b += (static_cast<uint32_t>(k[7]) << 24); [[fallthrough]]; + case 7 : b += (static_cast<uint32_t>(k[6]) << 16); [[fallthrough]]; + case 6 : b += (static_cast<uint32_t>(k[5]) << 8); [[fallthrough]]; + case 5 : b += k[4]; [[fallthrough]]; + case 4 : a += (static_cast<uint32_t>(k[3]) << 24); [[fallthrough]]; + case 3 : a += (static_cast<uint32_t>(k[2]) << 16); [[fallthrough]]; + case 2 : a += (static_cast<uint32_t>(k[1]) << 8); [[fallthrough]]; + case 1 : a += k[0]; /* case 0: nothing left to add */ } bobhash_mix(a,b,c); diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 111dfa071bb..a0963b43c34 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -116,10 +116,10 @@ main(int argc, char **argv) break; case 'r': doResetLevels = true; - //@fallthrough@ + [[fallthrough]]; case 'c': shouldCreateFile = true; - //@fallthrough@ + [[fallthrough]]; case 'n': shouldCreateEntry = true; break; diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index d43d779217d..ac7fd300828 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(vespamalloc $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS + atomic dl ) vespa_add_library(vespamallocd @@ -13,6 +14,7 @@ vespa_add_library(vespamallocd $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS + atomic dl ) vespa_add_library(vespamallocdst16 @@ -21,6 +23,7 @@ vespa_add_library(vespamallocdst16 $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS + atomic dl ) vespa_add_library(vespamallocdst16_nl @@ -29,6 +32,7 @@ vespa_add_library(vespamallocdst16_nl $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS + atomic dl ) vespa_add_library(vespammap @@ -36,4 +40,5 @@ vespa_add_library(vespammap $<TARGET_OBJECTS:vespamalloc_mmap> INSTALL lib64/vespa/malloc DEPENDS + dl ) diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index bd57c05f8ac..21fa1d9ed02 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -197,7 +197,7 @@ DocsumFilter::getFieldValue(const DocsumFieldSpec::FieldIdentifier & fieldId, return _cachedValue.get(); } } - //@fallthrough@ + [[fallthrough]]; default: return fv; } diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index 8855923610c..b21177a6810 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -65,7 +65,7 @@ FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & switch(searchDef) { default: LOG(warning, "Unknown searchdef = %d. Defaulting to AUTOUTF8", searchDef); - //@fallthrough@ + [[fallthrough]]; case VsmfieldsConfig::Fieldspec::AUTOUTF8: case VsmfieldsConfig::Fieldspec::NONE: case VsmfieldsConfig::Fieldspec::SSE2UTF8: diff --git a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java index 83c381e586f..82677a14242 100644 --- a/yolean/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/yolean/src/main/java/com/yahoo/yolean/Exceptions.java @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.yolean; +import java.io.IOException; +import java.io.UncheckedIOException; + /** * Helper methods for handling exceptions * @@ -44,4 +47,59 @@ public class Exceptions { return message; } + /** + * Wraps any IOException thrown from a runnable in an UncheckedIOException. + */ + public static void uncheck(RunnableThrowingIOException runnable) { + try { + runnable.run(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Wraps any IOException thrown from a runnable in an UncheckedIOException w/message. + */ + public static void uncheck(RunnableThrowingIOException runnable, String format, String... args) { + try { + runnable.run(); + } catch (IOException e) { + String message = String.format(format, (Object[]) args); + throw new UncheckedIOException(message, e); + } + } + + @FunctionalInterface + public interface RunnableThrowingIOException { + void run() throws IOException; + } + + /** + * Wraps any IOException thrown from a supplier in an UncheckedIOException. + */ + public static <T> T uncheck(SupplierThrowingIOException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Wraps any IOException thrown from a supplier in an UncheckedIOException w/message. + */ + public static <T> T uncheck(SupplierThrowingIOException<T> supplier, String format, String... args) { + try { + return supplier.get(); + } catch (IOException e) { + String message = String.format(format, (Object[]) args); + throw new UncheckedIOException(message, e); + } + } + + @FunctionalInterface + public interface SupplierThrowingIOException<T> { + T get() throws IOException; + } } diff --git a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java index db605609926..31e27fa2675 100644 --- a/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java +++ b/yolean/src/test/java/com/yahoo/yolean/ExceptionsTestCase.java @@ -3,6 +3,9 @@ package com.yahoo.yolean; import org.junit.Test; +import java.io.IOException; +import java.io.UncheckedIOException; + import static org.junit.Assert.assertEquals; /** @@ -21,4 +24,38 @@ public class ExceptionsTestCase { assertEquals("Foo",Exceptions.toMessageString(new Exception(new Exception("Foo")))); } + @Test + public void testUnchecks() { + try { + Exceptions.uncheck(this::throwIO); + } catch (UncheckedIOException e) { + assertEquals("root cause", e.getCause().getMessage()); + } + + try { + Exceptions.uncheck(this::throwIO, "additional %s", "info"); + } catch (UncheckedIOException e) { + assertEquals("additional info", e.getMessage()); + } + + try { + int i = Exceptions.uncheck(this::throwIOWithReturnValue); + } catch (UncheckedIOException e) { + assertEquals("root cause", e.getCause().getMessage()); + } + + try { + int i = Exceptions.uncheck(this::throwIOWithReturnValue, "additional %s", "info"); + } catch (UncheckedIOException e) { + assertEquals("additional info", e.getMessage()); + } + } + + private void throwIO() throws IOException { + throw new IOException("root cause"); + } + + private int throwIOWithReturnValue() throws IOException { + throw new IOException("root cause"); + } } |